cli: CommandAdd: Implement --no-content and --fill-content

cli/commandadd.cpp

@@ -44,20 +44,30 @@ int CommandAdd::handle(QStringList arguments)
 						"Continue adding files, don't exit on first error. If this option is not given, looqs will "
 						"exit asap, but it's possible that a few files will still be processed. "
 						"Set -t 1 to avoid this behavior, but processing will be slower. "},
+					   {{"n", "no-content"}, "Only add paths to database. Do not index content"},
+					   {{"f", "fill-content"}, "Index content for files previously indexed with -n"},
+					   {"tags", "Comma-separated list of tags to assign"},
 					   {{"t", "threads"}, "Number of threads to use.", "threads"}});
-
 	parser.addHelpOption();
 	parser.addPositionalArgument("add", "Add paths to the index",
 								 "add [paths...]. If no path is given, read from stdin, one path per line.");
 
 	parser.process(arguments);
 	this->keepGoing = parser.isSet("continue");
+	bool pathsOnly = parser.isSet("no-content");
+	bool fillContent = parser.isSet("fill-content");
 	if(parser.isSet("threads"))
 	{
 		QString threadsCount = parser.value("threads");
 		QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
 	}
 
+	if(pathsOnly && fillContent)
+	{
+		Logger::error() << "Invalid options: -n and -f cannot both be set";
+		return EXIT_FAILURE;
+	}
+
 	QStringList files = parser.positionalArguments();
 
 	if(files.length() == 0)

shared/filesaver.cpp

@@ -119,11 +119,29 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 		{
 			if(canonicalPath.startsWith(excludedPath))
 			{
+				if(this->fileSaverOptions.verbose)
+				{
+					Logger::info() << "Skipped due to excluded path";
+				}
 				return SKIPPED;
 			}
 		}
 
-		if(fileInfo.size() > 0)
+		bool mustFillContent = this->fileSaverOptions.fillPathsOnlyWithContent;
+		if(!mustFillContent)
+		{
+			mustFillContent = !this->fileSaverOptions.pathsOnly;
+			if(mustFillContent)
+			{
+				auto filetype = this->dbService->queryFileType(fileInfo.absolutePath());
+				if(filetype)
+				{
+					mustFillContent = filetype.value() == 'c';
+				}
+			}
+		}
+
+		if(fileInfo.size() > 0 && mustFillContent)
 		{
 			QProcess process;
 			QStringList args;
@@ -158,7 +176,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 			}
 		}
 	}
-	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData);
+	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.pathsOnly);
 	if(result == OK && processorReturnCode == OK_WASEMPTY)
 	{
 		return OK_WASEMPTY;

shared/filescanworker.cpp

@@ -12,6 +12,7 @@ FileScanWorker::FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &qu
 void FileScanWorker::run()
 {
 	FileSaver saver{*this->dbService};
+	saver.setFileSaverOptions(this->fileSaverOptions);
 	auto paths = queue->dequeue(batchsize);
 	for(QString &path : paths)
 	{
@@ -34,3 +35,8 @@ void FileScanWorker::run()
 	}
 	emit finished();
 }
+
+void FileScanWorker::setFileSaverOptions(FileSaverOptions options)
+{
+	this->fileSaverOptions = options;
+}

shared/filescanworker.h

@@ -15,12 +15,14 @@ class FileScanWorker : public QObject, public QRunnable
   protected:
 	SqliteDbService *dbService;
 	ConcurrentQueue<QString> *queue;
+	FileSaverOptions fileSaverOptions;
 	int batchsize;
 	std::atomic<bool> *stopToken;
 
   public:
 	FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &queue, int batchsize, std::atomic<bool> &stopToken);
 	void run() override;
+	void setFileSaverOptions(FileSaverOptions options);
   signals:
 	void result(FileScanResult);
 	void finished();

shared/indexer.cpp

@@ -98,6 +98,7 @@ void Indexer::launchWorker(ConcurrentQueue<QString> &queue, int batchsize)
 	FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize, this->workerCancellationToken);
 	connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult);
 	connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker);
+	runnable->setFileSaverOptions(this->fileSaverOptions);
 	++this->runningWorkers;
 	QThreadPool::globalInstance()->start(runnable);
 }

shared/sqlitedbservice.cpp

@@ -164,11 +164,15 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid
 	return true;
 }
 
-SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData)
+SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly)
 {
 	QString absPath = fileInfo.absoluteFilePath();
 	auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
-	QChar fileType = fileInfo.isDir() ? 'd' : 'f';
+	QChar fileType = fileInfo.isDir() ? 'd' : 'c';
+	if(pathsOnly)
+	{
+		fileType = 'f';
+	}
 
 	QSqlDatabase db = dbFactory->forCurrentThread();
 	QSqlQuery delQuery(db);
@@ -202,19 +206,23 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &
 		return DBFAIL;
 	}
 
-	int lastid = inserterQuery.lastInsertId().toInt();
-	if(!insertToFTS(false, db, lastid, pageData))
+	if(!pathsOnly)
 	{
-		db.rollback();
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
-		return DBFAIL;
-	}
-	if(!insertToFTS(true, db, lastid, pageData))
-	{
-		db.rollback();
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
-		return DBFAIL;
+		int lastid = inserterQuery.lastInsertId().toInt();
+		if(!insertToFTS(false, db, lastid, pageData))
+		{
+			db.rollback();
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+			return DBFAIL;
+		}
+		if(!insertToFTS(true, db, lastid, pageData))
+		{
+			db.rollback();
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+			return DBFAIL;
+		}
 	}
+
 	if(!db.commit())
 	{
 		db.rollback();

shared/sqlitedbservice.h

@@ -19,7 +19,7 @@ class SqliteDbService
 
   public:
 	SqliteDbService(DatabaseFactory &dbFactory);
-	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData);
+	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly);
 	unsigned int getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit);
 	bool deleteFile(QString path);
 	bool fileExistsInDatabase(QString path);