diff --git a/cli/commandadd.cpp b/cli/commandadd.cpp index 40003be..f6108ef 100644 --- a/cli/commandadd.cpp +++ b/cli/commandadd.cpp @@ -44,20 +44,30 @@ int CommandAdd::handle(QStringList arguments) "Continue adding files, don't exit on first error. If this option is not given, looqs will " "exit asap, but it's possible that a few files will still be processed. " "Set -t 1 to avoid this behavior, but processing will be slower. "}, + {{"n", "no-content"}, "Only add paths to database. Do not index content"}, + {{"f", "fill-content"}, "Index content for files previously indexed with -n"}, + {"tags", "Comma-separated list of tags to assign"}, {{"t", "threads"}, "Number of threads to use.", "threads"}}); - parser.addHelpOption(); parser.addPositionalArgument("add", "Add paths to the index", "add [paths...]. If no path is given, read from stdin, one path per line."); parser.process(arguments); this->keepGoing = parser.isSet("continue"); + bool pathsOnly = parser.isSet("no-content"); + bool fillContent = parser.isSet("fill-content"); if(parser.isSet("threads")) { QString threadsCount = parser.value("threads"); QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt()); } + if(pathsOnly && fillContent) + { + Logger::error() << "Invalid options: -n and -f cannot both be set"; + return EXIT_FAILURE; + } + QStringList files = parser.positionalArguments(); if(files.length() == 0) diff --git a/shared/filesaver.cpp b/shared/filesaver.cpp index 858de10..bb02ccb 100644 --- a/shared/filesaver.cpp +++ b/shared/filesaver.cpp @@ -119,11 +119,29 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) { if(canonicalPath.startsWith(excludedPath)) { + if(this->fileSaverOptions.verbose) + { + Logger::info() << "Skipped due to excluded path"; + } return SKIPPED; } } - if(fileInfo.size() > 0) + bool mustFillContent = this->fileSaverOptions.fillPathsOnlyWithContent; + if(!mustFillContent) + { + mustFillContent = !this->fileSaverOptions.pathsOnly; + if(mustFillContent) + { + auto filetype = this->dbService->queryFileType(fileInfo.absolutePath()); + if(filetype) + { + mustFillContent = filetype.value() == 'c'; + } + } + } + + if(fileInfo.size() > 0 && mustFillContent) { QProcess process; QStringList args; @@ -158,7 +176,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) } } } - SaveFileResult result = this->dbService->saveFile(fileInfo, pageData); + SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.pathsOnly); if(result == OK && processorReturnCode == OK_WASEMPTY) { return OK_WASEMPTY; diff --git a/shared/filescanworker.cpp b/shared/filescanworker.cpp index 47059ef..a33b41f 100644 --- a/shared/filescanworker.cpp +++ b/shared/filescanworker.cpp @@ -12,6 +12,7 @@ FileScanWorker::FileScanWorker(SqliteDbService &db, ConcurrentQueue &qu void FileScanWorker::run() { FileSaver saver{*this->dbService}; + saver.setFileSaverOptions(this->fileSaverOptions); auto paths = queue->dequeue(batchsize); for(QString &path : paths) { @@ -34,3 +35,8 @@ void FileScanWorker::run() } emit finished(); } + +void FileScanWorker::setFileSaverOptions(FileSaverOptions options) +{ + this->fileSaverOptions = options; +} diff --git a/shared/filescanworker.h b/shared/filescanworker.h index b7559d2..ce9d8cf 100644 --- a/shared/filescanworker.h +++ b/shared/filescanworker.h @@ -15,12 +15,14 @@ class FileScanWorker : public QObject, public QRunnable protected: SqliteDbService *dbService; ConcurrentQueue *queue; + FileSaverOptions fileSaverOptions; int batchsize; std::atomic *stopToken; public: FileScanWorker(SqliteDbService &db, ConcurrentQueue &queue, int batchsize, std::atomic &stopToken); void run() override; + void setFileSaverOptions(FileSaverOptions options); signals: void result(FileScanResult); void finished(); diff --git a/shared/indexer.cpp b/shared/indexer.cpp index 5cc33b2..e92ce4b 100644 --- a/shared/indexer.cpp +++ b/shared/indexer.cpp @@ -98,6 +98,7 @@ void Indexer::launchWorker(ConcurrentQueue &queue, int batchsize) FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize, this->workerCancellationToken); connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult); connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker); + runnable->setFileSaverOptions(this->fileSaverOptions); ++this->runningWorkers; QThreadPool::globalInstance()->start(runnable); } diff --git a/shared/sqlitedbservice.cpp b/shared/sqlitedbservice.cpp index 99f4224..f4075bb 100644 --- a/shared/sqlitedbservice.cpp +++ b/shared/sqlitedbservice.cpp @@ -164,11 +164,15 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid return true; } -SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector &pageData) +SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector &pageData, bool pathsOnly) { QString absPath = fileInfo.absoluteFilePath(); auto mtime = fileInfo.lastModified().toSecsSinceEpoch(); - QChar fileType = fileInfo.isDir() ? 'd' : 'f'; + QChar fileType = fileInfo.isDir() ? 'd' : 'c'; + if(pathsOnly) + { + fileType = 'f'; + } QSqlDatabase db = dbFactory->forCurrentThread(); QSqlQuery delQuery(db); @@ -202,19 +206,23 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector & return DBFAIL; } - int lastid = inserterQuery.lastInsertId().toInt(); - if(!insertToFTS(false, db, lastid, pageData)) + if(!pathsOnly) { - db.rollback(); - Logger::error() << "Failed to insert data to FTS index " << Qt::endl; - return DBFAIL; - } - if(!insertToFTS(true, db, lastid, pageData)) - { - db.rollback(); - Logger::error() << "Failed to insert data to FTS index " << Qt::endl; - return DBFAIL; + int lastid = inserterQuery.lastInsertId().toInt(); + if(!insertToFTS(false, db, lastid, pageData)) + { + db.rollback(); + Logger::error() << "Failed to insert data to FTS index " << Qt::endl; + return DBFAIL; + } + if(!insertToFTS(true, db, lastid, pageData)) + { + db.rollback(); + Logger::error() << "Failed to insert data to FTS index " << Qt::endl; + return DBFAIL; + } } + if(!db.commit()) { db.rollback(); diff --git a/shared/sqlitedbservice.h b/shared/sqlitedbservice.h index 1c9d09d..8682166 100644 --- a/shared/sqlitedbservice.h +++ b/shared/sqlitedbservice.h @@ -19,7 +19,7 @@ class SqliteDbService public: SqliteDbService(DatabaseFactory &dbFactory); - SaveFileResult saveFile(QFileInfo fileInfo, QVector &pageData); + SaveFileResult saveFile(QFileInfo fileInfo, QVector &pageData, bool pathsOnly); unsigned int getFiles(QVector &results, QString wildCardPattern, int offset, int limit); bool deleteFile(QString path); bool fileExistsInDatabase(QString path);