diff --git a/cli/commandadd.cpp b/cli/commandadd.cpp index b2f2079..c737a1a 100644 --- a/cli/commandadd.cpp +++ b/cli/commandadd.cpp @@ -10,109 +10,8 @@ #include #include #include -#include "processor.h" -#include "pdfprocessor.h" #include "commandadd.h" -#include "defaulttextprocessor.h" -#include "tagstripperprocessor.h" -#include "nothingprocessor.h" -#include "odtprocessor.h" -#include "odsprocessor.h" -#include "utils.h" -static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor(); -static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor(); -static NothingProcessor *nothingProcessor = new NothingProcessor(); -static OdtProcessor *odtProcessor = new OdtProcessor(); -static OdsProcessor *odsProcessor = new OdsProcessor(); - -static QMap processors{ - {"pdf", new PdfProcessor()}, {"txt", defaultTextProcessor}, {"md", defaultTextProcessor}, - {"py", defaultTextProcessor}, {"xml", nothingProcessor}, {"html", tagStripperProcessor}, - {"java", defaultTextProcessor}, {"js", defaultTextProcessor}, {"cpp", defaultTextProcessor}, - {"c", defaultTextProcessor}, {"sql", defaultTextProcessor}, {"odt", odtProcessor}, - {"ods", odsProcessor}}; - -AddFileResult CommandAdd::addFile(QString path) -{ - QFileInfo info(path); - QString absPath = info.absoluteFilePath(); - auto mtime = info.lastModified().toSecsSinceEpoch(); - QChar fileType = info.isDir() ? 'd' : 'f'; - - QSqlDatabase db = dbConnection(); - if(fileExistsInDatabase(db, absPath, mtime)) - { - return SKIPPED; - } - - Processor *processor = processors.value(info.suffix(), nothingProcessor); - QVector pageData; - if(processor->PREFERED_DATA_SOURCE == FILEPATH) - { - pageData = processor->process(absPath); - } - else - { - pageData = processor->process(Utils::readFile(absPath)); - } - - // Could happen if a file corrupted for example - if(pageData.isEmpty() && processor != nothingProcessor) - { - Utils::error() << "Could not get any content for " << absPath << endl; - } - - // Workaround to "database is locked" error. Perhaps try WAL mode etc. - // QMutexLocker locker(&writeMutex); - if(!db.transaction()) - { - Utils::error() << "Failed to open transaction for " << absPath << " : " << db.lastError() << endl; - return DBFAIL; - } - - QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db); - delQuery.addBindValue(absPath); - if(!delQuery.exec()) - { - Utils::error() << "Failed DELETE query" << delQuery.lastError() << endl; - db.rollback(); - return DBFAIL; - } - - QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db); - inserterQuery.addBindValue(absPath); - inserterQuery.addBindValue(mtime); - inserterQuery.addBindValue(info.size()); - inserterQuery.addBindValue(fileType); - if(!inserterQuery.exec()) - { - Utils::error() << "Failed INSERT query" << inserterQuery.lastError() << endl; - db.rollback(); - return DBFAIL; - } - int lastid = inserterQuery.lastInsertId().toInt(); - for(PageData &data : pageData) - { - QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db); - contentQuery.addBindValue(lastid); - contentQuery.addBindValue(data.pagenumber); - contentQuery.addBindValue(data.content); - if(!contentQuery.exec()) - { - db.rollback(); - Utils::error() << "Failed content insertion " << contentQuery.lastError() << endl; - return DBFAIL; - } - } - - if(!db.commit()) - { - db.rollback(); - Utils::error() << "Failed to commit transaction for " << absPath << " : " << db.lastError() << endl; - return DBFAIL; - } - return OK; -} +#include "logger.h" int CommandAdd::handle(QStringList arguments) { @@ -120,22 +19,14 @@ int CommandAdd::handle(QStringList arguments) parser.addOptions({{{"c", "continue"}, "Continue adding files, don't exit on first error"}, {{"a", "all"}, "On error, no files should be added, even already processed ones"}, {{"v", "verbose"}, "Print skipped and added files"}, - {{"n", "threads"}, "Number of threads to use.", "threads"}}); + {{"t", "threads"}, "Number of threads to use.", "threads"}}); parser.addHelpOption(); parser.addPositionalArgument("add", "Add paths to the index", "add [paths...]"); parser.process(arguments); - bool keepGoing = false; - bool verbose = false; - if(parser.isSet("continue")) - { - keepGoing = true; - } - if(parser.isSet("verbose")) - { - verbose = true; - } + bool keepGoing = parser.isSet("continue"); + bool verbose = parser.isSet("verbose"); if(parser.isSet("all")) { throw QSSGeneralException("To be implemented"); @@ -159,40 +50,12 @@ int CommandAdd::handle(QStringList arguments) } } - bool terminate = false; - QtConcurrent::blockingMap(files, - [&](QString &path) - { - if(terminate) - { - return; - } - if(verbose) - { - Utils::info() << "Processing " << path << endl; - } - auto result = addFile(path); - if(result == DBFAIL) - { - Utils::error() << "Failed to add " << path << endl; - if(!keepGoing) - { - terminate = true; - } - } - if(verbose) - { - if(result == SKIPPED) - { - Utils::info() - << "Skipped" << path << "as it already exists in the database" << endl; - } - else - { - Utils::info() << "Added" << path << endl; - } - } - }); + FileSaver saver(*this->dbService); + if(!saver.addFiles(files.toVector(), keepGoing, verbose)) + { + Logger::error() << "Errors occured while trying to add files to the database" << endl; + return 1; + } return 0; }