#include #include #include #include #include "filesaver.h" #include "processor.h" #include "pdfprocessor.h" #include "commandadd.h" #include "defaulttextprocessor.h" #include "tagstripperprocessor.h" #include "nothingprocessor.h" #include "odtprocessor.h" #include "odsprocessor.h" #include "utils.h" #include "logger.h" static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor(); static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor(); static NothingProcessor *nothingProcessor = new NothingProcessor(); static OdtProcessor *odtProcessor = new OdtProcessor(); static OdsProcessor *odsProcessor = new OdsProcessor(); static QMap processors{ {"pdf", new PdfProcessor() }, {"txt", defaultTextProcessor}, {"md", defaultTextProcessor}, {"py", defaultTextProcessor}, {"xml", nothingProcessor}, {"html", tagStripperProcessor}, {"java", defaultTextProcessor}, {"js", defaultTextProcessor}, {"cpp", defaultTextProcessor}, {"c", defaultTextProcessor}, {"sql", defaultTextProcessor}, {"odt", odtProcessor}, {"ods", odsProcessor} }; FileSaver::FileSaver(SqliteDbService &dbService) { this->dbService = &dbService; } SaveFileResult FileSaver::addFile(QString path) { QFileInfo info(path); QString absPath = info.absoluteFilePath(); auto mtime = info.lastModified().toSecsSinceEpoch(); if(this->dbService->fileExistsInDatabase(absPath, mtime)) { return SKIPPED; } return saveFile(info); } SaveFileResult FileSaver::updateFile(QString path) { QFileInfo info(path); return saveFile(info); } int FileSaver::addFiles(const QVector paths, bool keepGoing, bool verbose) { return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose); } int FileSaver::updateFiles(const QVector paths, bool keepGoing, bool verbose) { return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose); } int FileSaver::processFiles(const QVector paths, std::function saverFunc, bool keepGoing, bool verbose) { std::atomic terminate { false }; std::atomic processedCount { 0 }; QtConcurrent::blockingMap(paths, [&](const QString &path) { if(terminate.load()) { return; } if(verbose) { Logger::info() << "Processing " << path << endl; } SaveFileResult result = saverFunc(path); if(result == DBFAIL || result == PROCESSFAIL) { Logger::error() << "Failed to process " << path << endl; if(!keepGoing) { terminate = true; } } else { ++processedCount; if(verbose) { if(result == SKIPPED) { Logger::info() << "Skipped" << path << "as it already exists in the database" << endl; } else if(result == OK) { Logger::info() << "Added" << path << endl; } } } }); return processedCount.load(); } SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) { Processor *processor = processors.value(fileInfo.suffix(), nothingProcessor); QVector pageData; QString absPath = fileInfo.absoluteFilePath(); if(fileInfo.isFile()) { try { if(processor->PREFERED_DATA_SOURCE == FILEPATH) { pageData = processor->process(absPath); } else { pageData = processor->process(Utils::readFile(absPath)); } } catch(LooqsGeneralException &e) { Logger::error() << "Error while processing" << absPath << ":" << e.message << endl; return PROCESSFAIL; } } //Could happen if a file corrupted for example if(pageData.isEmpty() && processor != nothingProcessor) { Logger::error() << "Could not get any content for " << absPath << endl; } return this->dbService->saveFile(fileInfo, pageData); }