From bddacbedb8f81b7f6de4958e7a71782a0f390371 Mon Sep 17 00:00:00 2001 From: Albert S Date: Tue, 16 Apr 2019 08:52:43 +0200 Subject: [PATCH] CommandAdd: refactored to use filesaver etc. --- cli/commandadd.cpp | 169 +++------------------------------------------ 1 file changed, 11 insertions(+), 158 deletions(-) diff --git a/cli/commandadd.cpp b/cli/commandadd.cpp index 5cd953f..22ebb38 100644 --- a/cli/commandadd.cpp +++ b/cli/commandadd.cpp @@ -10,124 +10,11 @@ #include #include #include -#include "processor.h" -#include "pdfprocessor.h" #include "commandadd.h" -#include "defaulttextprocessor.h" -#include "tagstripperprocessor.h" -#include "nothingprocessor.h" -#include "odtprocessor.h" -#include "odsprocessor.h" -#include "utils.h" -static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor(); -static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor(); -static NothingProcessor *nothingProcessor = new NothingProcessor(); -static OdtProcessor *odtProcessor = new OdtProcessor(); -static OdsProcessor *odsProcessor = new OdsProcessor(); - -static QMap processors{ - {"pdf", new PdfProcessor() }, - {"txt", defaultTextProcessor}, - {"md", defaultTextProcessor}, - {"py", defaultTextProcessor}, - {"xml", nothingProcessor}, - {"html", tagStripperProcessor}, - {"java", defaultTextProcessor}, - {"js", defaultTextProcessor}, - {"cpp", defaultTextProcessor}, - {"c", defaultTextProcessor}, - {"sql", defaultTextProcessor}, - {"odt", odtProcessor}, - {"ods", odsProcessor} -}; +#include "logger.h" -AddFileResult CommandAdd::addFile(QString path) -{ - QFileInfo info(path); - QString absPath = info.absoluteFilePath(); - auto mtime = info.lastModified().toSecsSinceEpoch(); - QChar fileType = info.isDir() ? 'd' : 'f'; - - QSqlDatabase db = dbConnection(); - if(fileExistsInDatabase(db, absPath, mtime)) - { - return SKIPPED; - } - - - - Processor *processor = processors.value(info.suffix(), nothingProcessor); - QVector pageData; - if(processor->PREFERED_DATA_SOURCE == FILEPATH) - { - pageData = processor->process(absPath); - } - else - { - pageData = processor->process(Utils::readFile(absPath)); - } - - //Could happen if a file corrupted for example - if(pageData.isEmpty() && processor != nothingProcessor) - { - Utils::error() << "Could not get any content for " << absPath << endl; - } - - - //Workaround to "database is locked" error. Perhaps try WAL mode etc. - //QMutexLocker locker(&writeMutex); - if(!db.transaction()) - { - Utils::error() << "Failed to open transaction for " << absPath << " : " << db.lastError() << endl; - return DBFAIL; - } - - QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db); - delQuery.addBindValue(absPath); - if(!delQuery.exec()) - { - Utils::error() << "Failed DELETE query" << delQuery.lastError() << endl; - db.rollback(); - return DBFAIL; - } - - - QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db); - inserterQuery.addBindValue(absPath); - inserterQuery.addBindValue(mtime); - inserterQuery.addBindValue(info.size()); - inserterQuery.addBindValue(fileType); - if(!inserterQuery.exec()) - { - Utils::error() << "Failed INSERT query" << inserterQuery.lastError() << endl; - db.rollback(); - return DBFAIL; - } - int lastid = inserterQuery.lastInsertId().toInt(); - for(PageData &data : pageData) - { - QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db); - contentQuery.addBindValue(lastid); - contentQuery.addBindValue(data.pagenumber); - contentQuery.addBindValue(data.content); - if(!contentQuery.exec()) - { - db.rollback(); - Utils::error() << "Failed content insertion " << contentQuery.lastError() << endl; - return DBFAIL; - } - } - - if(!db.commit()) - { - db.rollback(); - Utils::error() << "Failed to commit transaction for " << absPath << " : " << db.lastError() << endl; - return DBFAIL; - } - return OK; -} int CommandAdd::handle(QStringList arguments) @@ -137,23 +24,15 @@ int CommandAdd::handle(QStringList arguments) { { "c", "continue" }, "Continue adding files, don't exit on first error"}, { { "a", "all" }, "On error, no files should be added, even already processed ones" }, { { "v", "verbose" }, "Print skipped and added files" }, - { { "n", "threads" }, "Number of threads to use.", "threads" } + { { "t", "threads" }, "Number of threads to use.", "threads" } }); parser.addHelpOption(); parser.addPositionalArgument("add", "Add paths to the index", "add [paths...]"); parser.process(arguments); - bool keepGoing = false; - bool verbose = false; - if(parser.isSet("continue")) - { - keepGoing = true; - } - if(parser.isSet("verbose")) - { - verbose = true; - } + bool keepGoing = parser.isSet("continue"); + bool verbose = parser.isSet("verbose"); if(parser.isSet("all")) { throw QSSGeneralException("To be implemented"); @@ -177,39 +56,13 @@ int CommandAdd::handle(QStringList arguments) } } - bool terminate = false; - QtConcurrent::blockingMap(files, [&](QString &path) { - if(terminate) - { - return; - } - if(verbose) - { - Utils::info() << "Processing " << path << endl; - } - auto result = addFile(path); - if(result == DBFAIL) - { - Utils::error() << "Failed to add " << path << endl; - if(!keepGoing) - { - terminate = true; - } - } - if(verbose) - { - if(result == SKIPPED) - { - Utils::info() << "Skipped" << path << "as it already exists in the database" << endl; - } - else - { - Utils::info() << "Added" << path << endl; - } - } - - - }); + FileSaver saver(*this->dbService); + if(!saver.addFiles(files.toVector(), keepGoing, verbose)) + { + Logger::error() << "Errors occured while trying to add files to the database" << endl; + return 1; + } return 0; } +