#include #include #include #include #include #include #include #include #include #include #include #include #include "processor.h" #include "pdfprocessor.h" #include "commandadd.h" #include "defaulttextprocessor.h" #include "tagstripperprocessor.h" #include "nothingprocessor.h" #include "odtprocessor.h" #include "odsprocessor.h" #include "utils.h" static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor(); static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor(); static NothingProcessor *nothingProcessor = new NothingProcessor(); static OdtProcessor *odtProcessor = new OdtProcessor(); static OdsProcessor *odsProcessor = new OdsProcessor(); static QMap processors{ {"pdf", new PdfProcessor()}, {"txt", defaultTextProcessor}, {"md", defaultTextProcessor}, {"py", defaultTextProcessor}, {"xml", nothingProcessor}, {"html", tagStripperProcessor}, {"java", defaultTextProcessor}, {"js", defaultTextProcessor}, {"cpp", defaultTextProcessor}, {"c", defaultTextProcessor}, {"sql", defaultTextProcessor}, {"odt", odtProcessor}, {"ods", odsProcessor}}; AddFileResult CommandAdd::addFile(QString path) { QFileInfo info(path); QString absPath = info.absoluteFilePath(); auto mtime = info.lastModified().toSecsSinceEpoch(); QChar fileType = info.isDir() ? 'd' : 'f'; QSqlDatabase db = dbConnection(); if(fileExistsInDatabase(db, absPath, mtime)) { return SKIPPED; } Processor *processor = processors.value(info.suffix(), nothingProcessor); QVector pageData; if(processor->PREFERED_DATA_SOURCE == FILEPATH) { pageData = processor->process(absPath); } else { pageData = processor->process(Utils::readFile(absPath)); } if(pageData.isEmpty()) { qDebug() << "Could not get any content for " << absPath; } // Workaround to "database is locked" error. Perhaps try WAL mode etc. // QMutexLocker locker(&writeMutex); if(!db.transaction()) { qDebug() << "Failed to open transaction for " << absPath << " : " << db.lastError(); return DBFAIL; } QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db); delQuery.addBindValue(absPath); if(!delQuery.exec()) { qDebug() << "Failed DELETE query" << delQuery.lastError(); db.rollback(); return DBFAIL; } QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db); inserterQuery.addBindValue(absPath); inserterQuery.addBindValue(mtime); inserterQuery.addBindValue(info.size()); inserterQuery.addBindValue(fileType); if(!inserterQuery.exec()) { qDebug() << "Failed INSERT query" << inserterQuery.lastError(); db.rollback(); return DBFAIL; } int lastid = inserterQuery.lastInsertId().toInt(); for(PageData &data : pageData) { QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db); contentQuery.addBindValue(lastid); contentQuery.addBindValue(data.pagenumber); contentQuery.addBindValue(data.content); if(!contentQuery.exec()) { db.rollback(); qDebug() << "Failed content insertion " << contentQuery.lastError(); return DBFAIL; } } if(!db.commit()) { db.rollback(); qDebug() << "Failed to commit transaction for " << absPath << " : " << db.lastError(); return DBFAIL; } return OK; } int CommandAdd::handle(QStringList arguments) { QCommandLineParser parser; parser.addOptions({{{"c", "continue"}, "Continue adding files, don't exit on first error"}, {{"a", "all"}, "On error, no files should be added, even already processed ones"}, {{"v", "verbose"}, "Print skipped and added files"}, {{"n", "threads"}, "Number of threads to use.", "threads"}}); parser.addHelpOption(); parser.addPositionalArgument("paths", "List of paths to process/add to the index", "[paths...]"); parser.process(arguments); bool keepGoing = false; bool verbose = false; if(parser.isSet("continue")) { keepGoing = true; } if(parser.isSet("verbose")) { verbose = true; } if(parser.isSet("all")) { throw QSSGeneralException("To be implemented"); } if(parser.isSet("threads")) { QString threadsCount = parser.value("threads"); QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt()); } QStringList files = parser.positionalArguments(); if(files.length() == 0) { QTextStream stream(stdin); while(!stream.atEnd()) { QString path = stream.readLine(); files.append(path); } } bool terminate = false; QtConcurrent::blockingMap(files, [&](QString &path) { if(terminate) { return; } if(verbose) { qDebug() << "Processing " << path; } auto result = addFile(path); if(result == DBFAIL) { qDebug() << "Failed to add " << path; if(!keepGoing) { terminate = true; } } if(verbose) { if(result == SKIPPED) { qDebug() << "SKIPPED" << path << "as it already exists in the database"; } else { qDebug() << "Added" << path; } } }); return 0; }