shared: sqlitedbservice: Introduce exec(),execBool(). Refactor

Apparently poppler or something needs open() with write
flags to render pdfs with proper fonts.

Landlock guards file system write access, so this is fine.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2018-2022: Albert Schwarzkopf <looqs at quitesimple period org>
+Copyright (c) 2018-2023: Albert Schwarzkopf <looqs at quitesimple period org>
 
 looqs is made available under the following license: 
 

cli/commandadd.cpp

@@ -23,7 +23,7 @@ void CommandAdd::indexerFinished()
 	if(failedPathsCount > 0)
 	{
 		Logger::info() << "Failed paths: " << Qt::endl;
-		for(QString paths : result.failedPaths())
+		for(const QString &paths : result.failedPaths())
 		{
 			Logger::info() << paths << Qt::endl;
 		}
@@ -44,20 +44,30 @@ int CommandAdd::handle(QStringList arguments)
 						"Continue adding files, don't exit on first error. If this option is not given, looqs will "
 						"exit asap, but it's possible that a few files will still be processed. "
 						"Set -t 1 to avoid this behavior, but processing will be slower. "},
+					   {{"n", "no-content"}, "Only add paths to database. Do not index content"},
+					   {{"f", "fill-content"}, "Index content for files previously indexed with -n"},
+					   {"tags", "Comma-separated list of tags to assign"},
 					   {{"t", "threads"}, "Number of threads to use.", "threads"}});
-
 	parser.addHelpOption();
 	parser.addPositionalArgument("add", "Add paths to the index",
 								 "add [paths...]. If no path is given, read from stdin, one path per line.");
 
 	parser.process(arguments);
 	this->keepGoing = parser.isSet("continue");
+	bool pathsOnly = parser.isSet("no-content");
+	bool fillContent = parser.isSet("fill-content");
 	if(parser.isSet("threads"))
 	{
 		QString threadsCount = parser.value("threads");
 		QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
 	}
 
+	if(pathsOnly && fillContent)
+	{
+		Logger::error() << "Invalid options: -n and -f cannot both be set";
+		return EXIT_FAILURE;
+	}
+
 	QStringList files = parser.positionalArguments();
 
 	if(files.length() == 0)
@@ -71,15 +81,22 @@ int CommandAdd::handle(QStringList arguments)
 		}
 	}
 
+	FileSaverOptions fileSaverOptions;
+	fileSaverOptions.keepGoing = keepGoing;
+	fileSaverOptions.fillExistingContentless = fillContent;
+	fileSaverOptions.metadataOnly = pathsOnly;
+	fileSaverOptions.verbose = false;
+
 	indexer = new Indexer(*this->dbService);
+	indexer->setFileSaverOptions(fileSaverOptions);
+
 	indexer->setTargetPaths(files.toVector());
-	indexer->setKeepGoing(keepGoing);
 
 	connect(indexer, &Indexer::pathsCountChanged, this,
 			[](int pathsCount) { Logger::info() << "Found paths: " << pathsCount << Qt::endl; });
 	connect(indexer, &Indexer::indexProgress, this,
-			[](int pathsCount, unsigned int added, unsigned int skipped, unsigned int failed, unsigned int totalCount)
-			{ Logger::info() << "Processed files: " << pathsCount << Qt::endl; });
+			[](int pathsCount, unsigned int /*added*/, unsigned int /*skipped*/, unsigned int /*failed*/,
+			   unsigned int /*totalCount*/) { Logger::info() << "Processed files: " << pathsCount << Qt::endl; });
 	connect(indexer, &Indexer::finished, this, &CommandAdd::indexerFinished);
 
 	this->autoFinish = false;

cli/commandlist.cpp

@@ -1,6 +1,5 @@
 #include <QCommandLineParser>
 #include "commandlist.h"
-#include "databasefactory.h"
 #include "logger.h"
 
 int CommandList::handle(QStringList arguments)

cli/commandupdate.cpp

@@ -38,10 +38,13 @@ int CommandUpdate::handle(QStringList arguments)
 		QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
 	}
 
-	bool hasErrors = false;
 	IndexSyncer *syncer = new IndexSyncer(*this->dbService);
-	syncer->setKeepGoing(keepGoing);
-	syncer->setVerbose(verbose);
+
+	FileSaverOptions fileOptions;
+	fileOptions.keepGoing = keepGoing;
+	fileOptions.verbose = verbose;
+
+	syncer->setFileSaverOptions(fileOptions);
 	syncer->setPattern(pattern);
 	syncer->setDryRun(dryRun);
 	syncer->setRemoveDeletedFromIndex(deleteMissing);
@@ -60,7 +63,7 @@ int CommandUpdate::handle(QStringList arguments)
 		/* TODO: updated not printed, handled be verbose in FileSaver, but this can be improved */
 	}
 	connect(syncer, &IndexSyncer::finished, this,
-			[&](unsigned int totalUpdated, unsigned int totalRemoved, unsigned int totalErrors)
+			[this, dryRun, keepGoing](unsigned int totalUpdated, unsigned int totalRemoved, unsigned int totalErrors)
 			{
 				Logger::info() << "Syncing finished" << Qt::endl;
 
@@ -72,7 +75,7 @@ int CommandUpdate::handle(QStringList arguments)
 				}
 
 				int retval = 0;
-				if(hasErrors && !keepGoing)
+				if(this->hasErrors && !keepGoing)
 				{
 					retval = 1;
 				}
@@ -82,7 +85,7 @@ int CommandUpdate::handle(QStringList arguments)
 			[&](QString error)
 			{
 				Logger::error() << error << Qt::endl;
-				hasErrors = true;
+				this->hasErrors = true;
 			});
 
 	this->autoFinish = false;

cli/commandupdate.h

@@ -4,6 +4,9 @@
 #include "filesaver.h"
 class CommandUpdate : public Command
 {
+  protected:
+	bool hasErrors = false;
+
   public:
 	using Command::Command;
 	int handle(QStringList arguments) override;

gui/gui.pro

@@ -34,6 +34,7 @@ SOURCES += \
         main.cpp \
         mainwindow.cpp \
       clicklabel.cpp \
+    previewcoordinator.cpp \
     previewgenerator.cpp \
     previewgeneratormapfunctor.cpp \
     previewgeneratorodt.cpp \
@@ -54,6 +55,7 @@ HEADERS += \
     ipcserver.h \
         mainwindow.h \
     clicklabel.h \
+    previewcoordinator.h \
     previewgenerator.h \
     previewgeneratormapfunctor.h \
     previewgeneratorodt.h \

gui/main.cpp

@@ -28,7 +28,7 @@ void enableIpcSandbox()
 	policy->namespace_options = EXILE_UNSHARE_USER | EXILE_UNSHARE_MOUNT | EXILE_UNSHARE_NETWORK;
 	policy->no_new_privs = 1;
 	policy->drop_caps = 1;
-	policy->vow_promises = exile_vows_from_str("thread cpath rpath unix stdio proc error");
+	policy->vow_promises = exile_vows_from_str("thread cpath rpath wpath unix stdio proc error");
 	policy->mount_path_policies_to_chroot = 1;
 
 	QString ipcSocketPath = Common::ipcSocketPath();

gui/mainwindow.cpp

@@ -22,8 +22,6 @@
 #include "../shared/sqlitesearch.h"
 #include "../shared/looqsgeneralexception.h"
 #include "../shared/common.h"
-#include "ipcpreviewclient.h"
-#include "previewgenerator.h"
 #include "aboutdialog.h"
 
 MainWindow::MainWindow(QWidget *parent, QString socketPath)
@@ -32,8 +30,7 @@ MainWindow::MainWindow(QWidget *parent, QString socketPath)
 	this->progressDialog.cancel(); // because constructing it shows it, quite weird
 	ui->setupUi(this);
 	setWindowTitle(QCoreApplication::applicationName());
-	this->ipcPreviewClient.moveToThread(&this->ipcClientThread);
-	this->ipcPreviewClient.setSocketPath(socketPath);
+
 	QSettings settings;
 
 	this->dbFactory = new DatabaseFactory(Common::databasePath());
@@ -78,7 +75,7 @@ MainWindow::MainWindow(QWidget *parent, QString socketPath)
 	ui->txtSearch->installEventFilter(this);
 	ui->scrollArea->viewport()->installEventFilter(this);
 
-	this->ipcClientThread.start();
+	this->previewCoordinator.setSocketPath(socketPath);
 }
 
 void MainWindow::addPathToIndex()
@@ -150,7 +147,7 @@ void MainWindow::connectSignals()
 	connect(this->indexer, &Indexer::finished, this, &MainWindow::finishIndexing);
 
 	connect(ui->lstPaths->selectionModel(), &QItemSelectionModel::selectionChanged, this,
-			[&](const QItemSelection &selected, const QItemSelection &deselected)
+			[&](const QItemSelection & /*selected*/, const QItemSelection & /*deselected*/)
 			{ ui->btnDeletePath->setEnabled(this->ui->lstPaths->selectedItems().count() > 0); });
 
 	connect(ui->btnDeletePath, &QPushButton::clicked, this, [&] { qDeleteAll(ui->lstPaths->selectedItems()); });
@@ -170,30 +167,29 @@ void MainWindow::connectSignals()
 				}
 			});
 	connect(ui->menuAboutAction, &QAction::triggered, this,
-			[this](bool checked)
+			[this](bool /*checked*/)
 			{
 				AboutDialog aboutDialog(this);
 
 				aboutDialog.exec();
 			});
 	connect(ui->menuAboutQtAction, &QAction::triggered, this,
-			[this](bool checked) { QMessageBox::aboutQt(this, "About Qt"); });
+			[this](bool /*checked*/) { QMessageBox::aboutQt(this, "About Qt"); });
 	connect(ui->menuSyncIndexAction, &QAction::triggered, this, &MainWindow::startIndexSync);
 	connect(ui->menuOpenUserManualAction, &QAction::triggered, this,
-			[this]() { QDesktopServices::openUrl(Common::userManualUrl()); });
+			[]() { QDesktopServices::openUrl(Common::userManualUrl()); });
 
-	connect(indexSyncer, &IndexSyncer::finished, this,
-			[&](unsigned int totalUpdated, unsigned int totalDeleted, unsigned int totalErrored)
-			{
-				this->progressDialog.cancel();
+	connect(
+		indexSyncer, &IndexSyncer::finished, this,
+		[&](unsigned int totalUpdated, unsigned int totalDeleted, unsigned int totalErrored)
+		{
+			this->progressDialog.cancel();
 
-				QMessageBox::information(
-					this, "Syncing finished",
-					QString("Syncing finished\n\nTotal updated: %1\nTotal deleted: %2\nTotal errors: %3\n")
-						.arg(QString::number(totalUpdated))
-						.arg(QString::number(totalDeleted))
-						.arg(QString::number(totalErrored)));
-			});
+			QMessageBox::information(
+				this, "Syncing finished",
+				QString("Syncing finished\n\nTotal updated: %1\nTotal deleted: %2\nTotal errors: %3\n")
+					.arg(QString::number(totalUpdated), QString::number(totalDeleted), QString::number(totalErrored)));
+		});
 	connect(this, &MainWindow::beginIndexSync, indexSyncer, &IndexSyncer::sync);
 	connect(&this->progressDialog, &QProgressDialog::canceled, indexSyncer, &IndexSyncer::cancel);
 	connect(ui->btnSaveSettings, &QPushButton::clicked, this, &MainWindow::saveSettings);
@@ -208,9 +204,9 @@ void MainWindow::connectSignals()
 			}
 		},
 		Qt::QueuedConnection);
-	connect(&ipcPreviewClient, &IPCPreviewClient::previewReceived, this, &MainWindow::previewReceived,
+	connect(&previewCoordinator, &PreviewCoordinator::previewReady, this, &MainWindow::previewReceived,
 			Qt::QueuedConnection);
-	connect(&ipcPreviewClient, &IPCPreviewClient::finished, this,
+	connect(&previewCoordinator, &PreviewCoordinator::completedGeneration, this,
 			[&]
 			{
 				this->ui->previewProcessBar->setValue(this->ui->previewProcessBar->maximum());
@@ -218,22 +214,24 @@ void MainWindow::connectSignals()
 				this->ui->comboPreviewFiles->setEnabled(true);
 				ui->txtSearch->setEnabled(true);
 			});
-	connect(&ipcPreviewClient, &IPCPreviewClient::error, this,
+	connect(&previewCoordinator, &PreviewCoordinator::error, this,
 			[this](QString msg)
 			{
 				qCritical() << msg << Qt::endl;
 				QMessageBox::critical(this, "IPC error", msg);
 			});
-
-	connect(this, &MainWindow::startIpcPreviews, &ipcPreviewClient, &IPCPreviewClient::startGeneration,
-			Qt::QueuedConnection);
-	connect(this, &MainWindow::stopIpcPreviews, &ipcPreviewClient, &IPCPreviewClient::stopGeneration,
-			Qt::QueuedConnection);
+	connect(ui->radioMetadataOnly, &QRadioButton::toggled, this,
+			[this](bool toggled)
+			{
+				if(toggled)
+				{
+					this->ui->chkFillContentForContentless->setChecked(false);
+				};
+			});
 }
 
 void MainWindow::exportFailedPaths()
 {
-
 	QString filename =
 		QString("/tmp/looqs_indexresult_failed_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_hhmmss"));
 	QFile outFile(filename);
@@ -266,8 +264,11 @@ void MainWindow::startIndexSync()
 	progressDialog.setValue(0);
 	progressDialog.open();
 
-	indexSyncer->setKeepGoing(true);
-	indexSyncer->setVerbose(false);
+	FileSaverOptions options;
+	options.keepGoing = true;
+	options.verbose = false;
+
+	indexSyncer->setFileSaverOptions(options);
 	indexSyncer->setDryRun(false);
 	indexSyncer->setRemoveDeletedFromIndex(true);
 
@@ -311,6 +312,15 @@ void MainWindow::startIndexing()
 	this->indexer->setTargetPaths(paths);
 	QString ignorePatterns = ui->txtIgnorePatterns->text();
 	this->indexer->setIgnorePattern(ignorePatterns.split(";"));
+
+	FileSaverOptions options;
+	options.fillExistingContentless =
+		ui->chkFillContentForContentless->isEnabled() && ui->chkFillContentForContentless->isChecked();
+	options.metadataOnly = ui->radioMetadataOnly->isChecked();
+	options.verbose = false;
+	options.keepGoing = true;
+
+	this->indexer->setFileSaverOptions(options);
 	this->indexer->beginIndexing();
 	QSettings settings;
 	settings.setValue("indexPaths", pathSettingsValue);
@@ -340,7 +350,7 @@ void MainWindow::finishIndexing()
 	}
 }
 
-void MainWindow::comboScaleChanged(int i)
+void MainWindow::comboScaleChanged(int /*i*/)
 {
 	QSettings scaleSetting;
 	scaleSetting.setValue("currentScale", ui->comboScale->currentText());
@@ -386,7 +396,8 @@ void MainWindow::processShortcut(int key)
 	{
 		ui->txtSearch->setFocus();
 		QString currentText = ui->txtSearch->text().trimmed();
-		int index = currentText.lastIndexOf(QRegularExpression("[\\s\\)]"));
+		static QRegularExpression separatorRegex("[\\s\\)]");
+		int index = currentText.lastIndexOf(separatorRegex);
 		if(index != -1)
 		{
 			bool isFilter = (index == currentText.length() - 1);
@@ -632,13 +643,17 @@ void MainWindow::saveSettings()
 	qApp->quit();
 }
 
-void MainWindow::previewReceived(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration)
+void MainWindow::previewReceived()
 {
-	if(previewGeneration < this->currentPreviewGeneration)
-	{
-		return;
-	}
 	this->ui->previewProcessBar->setValue(this->ui->previewProcessBar->value() + 1);
+	QBoxLayout *layout = static_cast<QBoxLayout *>(ui->scrollAreaWidgetContents->layout());
+	int index = layout->count();
+	if(index > 0)
+	{
+		--index;
+	}
+	QSharedPointer<PreviewResult> preview = this->previewCoordinator.resultAt(index);
+
 	if(!preview.isNull() && preview->hasPreview())
 	{
 		QString docPath = preview->getDocumentPath();
@@ -661,8 +676,8 @@ void MainWindow::previewReceived(QSharedPointer<PreviewResult> preview, unsigned
 		{
 			QFileInfo fileInfo{docPath};
 			QMenu menu("labeRightClick", this);
-			createSearchResutlMenu(menu, fileInfo);
-			menu.addAction("Copy page number",
+			createSearchResultMenu(menu, fileInfo);
+			menu.addAction("Copy page number", this,
 						   [previewPage] { QGuiApplication::clipboard()->setText(QString::number(previewPage)); });
 			menu.exec(QCursor::pos());
 		};
@@ -684,24 +699,7 @@ void MainWindow::previewReceived(QSharedPointer<PreviewResult> preview, unsigned
 
 		previewWidget->setLayout(previewLayout);
 
-		QBoxLayout *layout = static_cast<QBoxLayout *>(ui->scrollAreaWidgetContents->layout());
-		int pos = previewOrder[docPath + QString::number(previewPage)];
-		if(pos <= layout->count())
-		{
-			layout->insertWidget(pos, previewWidget);
-			for(auto it = previewWidgetOrderCache.constKeyValueBegin();
-				it != previewWidgetOrderCache.constKeyValueEnd(); it++)
-			{
-				if(it->first <= layout->count())
-				{
-					layout->insertWidget(it->first, it->second);
-				}
-			}
-		}
-		else
-		{
-			previewWidgetOrderCache[pos] = previewWidget;
-		}
+		layout->insertWidget(index, previewWidget);
 	}
 }
 
@@ -818,7 +816,6 @@ void MainWindow::lineEditReturnPressed()
 
 void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 {
-	this->previewableSearchResults.clear();
 	qDeleteAll(ui->scrollAreaWidgetContents->children());
 
 	ui->treeResultsList->clear();
@@ -827,6 +824,8 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 	ui->comboPreviewFiles->setVisible(true);
 	ui->lblTotalPreviewPagesCount->setText("");
 
+	this->previewCoordinator.init(results);
+
 	bool hasDeleted = false;
 	QHash<QString, bool> seenMap;
 	for(const SearchResult &result : results)
@@ -847,34 +846,29 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 			item->setText(3, this->locale().formattedDataSize(result.fileData.size));
 		}
 		bool exists = pathInfo.exists();
-		if(exists)
-		{
-			if(result.wasContentSearch)
-			{
-				if(!pathInfo.suffix().contains("htm")) // hack until we can preview them properly...
-				{
-					if(PreviewGenerator::get(pathInfo) != nullptr)
-					{
-						this->previewableSearchResults.append(result);
-						if(!seenMap.contains(result.fileData.absPath))
-						{
-							ui->comboPreviewFiles->addItem(result.fileData.absPath);
-						}
-					}
-				}
-			}
-		}
-		else
+		if(!exists)
 		{
 			hasDeleted = true;
 		}
 		seenMap[absPath] = true;
 	}
 
+	seenMap.clear();
+	for(const SearchResult &result : this->previewCoordinator.getPreviewableSearchResults())
+	{
+		const QString &absPath = result.fileData.absPath;
+		if(!seenMap.contains(absPath))
+		{
+			ui->comboPreviewFiles->addItem(absPath);
+		}
+		seenMap[absPath] = true;
+	}
+
 	ui->treeResultsList->resizeColumnToContents(0);
 	ui->treeResultsList->resizeColumnToContents(1);
 	ui->treeResultsList->resizeColumnToContents(2);
-	previewDirty = !this->previewableSearchResults.empty();
+
+	previewDirty = this->previewCoordinator.previewableCount() > 0;
 
 	ui->spinPreviewPage->setValue(1);
 
@@ -884,7 +878,7 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 	}
 
 	QString statusText = "Results: " + QString::number(results.size()) + " files";
-	statusText += ", previewable: " + QString::number(this->previewableSearchResults.count());
+	statusText += ", previewable: " + QString::number(this->previewCoordinator.previewableCount());
 	if(hasDeleted)
 	{
 		statusText += " WARNING: Some files are inaccessible. No preview available for those. Index may be out of sync";
@@ -901,7 +895,7 @@ int MainWindow::currentSelectedScale()
 
 void MainWindow::makePreviews(int page)
 {
-	if(this->previewableSearchResults.empty())
+	if(this->previewCoordinator.previewableCount() == 0)
 	{
 		return;
 	}
@@ -918,11 +912,10 @@ void MainWindow::makePreviews(int page)
 		ui->scrollAreaWidgetContents->setLayout(new QVBoxLayout());
 		ui->scrollAreaWidgetContents->layout()->setAlignment(Qt::AlignCenter);
 	}
-	ui->previewProcessBar->setMaximum(this->previewableSearchResults.size());
-	processedPdfPreviews = 0;
+	ui->previewProcessBar->setMaximum(this->previewCoordinator.previewableCount());
 
 	QVector<QString> wordsToHighlight;
-	QRegularExpression extractor(R"#("([^"]*)"|([^\s]+))#");
+	static QRegularExpression extractor(R"#("([^"]*)"|([^\s]+))#");
 	for(const Token &token : this->contentSearchQuery.getTokens())
 	{
 		if(token.type == FILTER_CONTENT_CONTAINS)
@@ -954,12 +947,8 @@ void MainWindow::makePreviews(int page)
 	renderConfig.scaleY = QGuiApplication::primaryScreen()->physicalDotsPerInchY() * (currentScale / 100.);
 	renderConfig.wordsToHighlight = wordsToHighlight;
 
-	this->previewOrder.clear();
-	this->previewWidgetOrderCache.clear();
-
-	int previewPos = 0;
 	QVector<RenderTarget> targets;
-	for(SearchResult &sr : this->previewableSearchResults)
+	for(const SearchResult &sr : this->previewCoordinator.getPreviewableSearchResults())
 	{
 		if(ui->comboPreviewFiles->currentIndex() != 0)
 		{
@@ -971,11 +960,8 @@ void MainWindow::makePreviews(int page)
 		RenderTarget renderTarget;
 		renderTarget.path = sr.fileData.absPath;
 		renderTarget.page = (int)sr.page;
-		targets.append(renderTarget);
 
-		int pos = previewPos - beginOffset;
-		this->previewOrder[renderTarget.path + QString::number(renderTarget.page)] = pos;
-		++previewPos;
+		targets.append(renderTarget);
 	}
 	int numpages = ceil(static_cast<double>(targets.size()) / previewsPerPage);
 	ui->spinPreviewPage->setMaximum(numpages);
@@ -985,12 +971,12 @@ void MainWindow::makePreviews(int page)
 	ui->previewProcessBar->setMaximum(targets.count());
 	ui->previewProcessBar->setMinimum(0);
 	ui->previewProcessBar->setValue(0);
-	ui->previewProcessBar->setVisible(this->previewableSearchResults.size() > 0);
-	++this->currentPreviewGeneration;
+	ui->previewProcessBar->setVisible(this->previewCoordinator.previewableCount() > 0);
 	this->ui->spinPreviewPage->setEnabled(false);
 	this->ui->comboPreviewFiles->setEnabled(false);
 	this->ui->txtSearch->setEnabled(false);
-	emit startIpcPreviews(renderConfig, targets);
+
+	this->previewCoordinator.startGeneration(renderConfig, targets);
 }
 
 void MainWindow::handleSearchError(QString error)
@@ -998,21 +984,22 @@ void MainWindow::handleSearchError(QString error)
 	ui->lblSearchResults->setText("Error:" + error);
 }
 
-void MainWindow::createSearchResutlMenu(QMenu &menu, const QFileInfo &fileInfo)
+void MainWindow::createSearchResultMenu(QMenu &menu, const QFileInfo &fileInfo)
 {
-	menu.addAction("Copy filename to clipboard",
+	menu.addAction("Copy filename to clipboard", this,
 				   [&fileInfo] { QGuiApplication::clipboard()->setText(fileInfo.fileName()); });
-	menu.addAction("Copy full path to clipboard",
+	menu.addAction("Copy full path to clipboard", this,
 				   [&fileInfo] { QGuiApplication::clipboard()->setText(fileInfo.absoluteFilePath()); });
-	menu.addAction("Open containing folder", [this, &fileInfo] { this->openFile(fileInfo.absolutePath()); });
+	menu.addAction("Open containing folder", this, [this, &fileInfo] { this->openFile(fileInfo.absolutePath()); });
 
+	auto previewables = this->previewCoordinator.getPreviewableSearchResults();
 	auto result =
-		std::find_if(this->previewableSearchResults.begin(), this->previewableSearchResults.end(),
-					 [this, &fileInfo](SearchResult &a) { return fileInfo.absoluteFilePath() == a.fileData.absPath; });
+		std::find_if(previewables.begin(), previewables.end(),
+					 [&fileInfo](SearchResult &a) { return fileInfo.absoluteFilePath() == a.fileData.absPath; });
 
-	if(result != this->previewableSearchResults.end())
+	if(result != previewables.end())
 	{
-		menu.addAction("Show previews for this file",
+		menu.addAction("Show previews for this file", this,
 					   [this, &fileInfo]
 					   {
 						   ui->tabWidget->setCurrentIndex(1);
@@ -1048,7 +1035,7 @@ void MainWindow::openFile(QString path)
 	QDesktopServices::openUrl(QUrl::fromLocalFile(path));
 }
 
-void MainWindow::treeSearchItemActivated(QTreeWidgetItem *item, int i)
+void MainWindow::treeSearchItemActivated(QTreeWidgetItem *item, int /*i*/)
 {
 	openFile(item->text(1));
 }
@@ -1062,14 +1049,13 @@ void MainWindow::showSearchResultsContextMenu(const QPoint &point)
 	}
 	QFileInfo pathinfo(item->text(1));
 	QMenu menu("SearchResults", this);
-	createSearchResutlMenu(menu, pathinfo);
+	createSearchResultMenu(menu, pathinfo);
 	menu.exec(QCursor::pos());
 }
 
 MainWindow::~MainWindow()
 {
 	syncerThread.terminate();
-	ipcClientThread.terminate();
 	delete this->indexSyncer;
 	delete this->dbService;
 	delete this->dbFactory;
@@ -1077,7 +1063,7 @@ MainWindow::~MainWindow()
 	delete ui;
 }
 
-void MainWindow::closeEvent(QCloseEvent *event)
+void MainWindow::closeEvent(QCloseEvent * /*event*/)
 {
 	QStringList list = this->searchHistory.toList();
 	QSettings settings;

gui/mainwindow.h

@@ -12,7 +12,7 @@
 #include <QProgressDialog>
 #include "../shared/looqsquery.h"
 #include "../shared/indexsyncer.h"
-#include "ipcpreviewclient.h"
+#include "previewcoordinator.h"
 #include "indexer.h"
 namespace Ui
 {
@@ -27,8 +27,9 @@ class MainWindow : public QMainWindow
 	DatabaseFactory *dbFactory;
 	SqliteDbService *dbService;
 	Ui::MainWindow *ui;
-	IPCPreviewClient ipcPreviewClient;
-	QThread ipcClientThread;
+
+	PreviewCoordinator previewCoordinator;
+
 	QThread syncerThread;
 	Indexer *indexer;
 	IndexSyncer *indexSyncer;
@@ -36,18 +37,12 @@ class MainWindow : public QMainWindow
 	QFileIconProvider iconProvider;
 	QSqlDatabase db;
 	QFutureWatcher<QVector<SearchResult>> searchWatcher;
-	QVector<SearchResult> previewableSearchResults;
 	LooqsQuery contentSearchQuery;
 	QVector<QString> searchHistory;
 	int currentSearchHistoryIndex = 0;
 	QString currentSavedSearchText;
-	QHash<QString, int> previewOrder; /* Quick lookup for the order a preview should have */
-	QMap<int, QWidget *>
-		previewWidgetOrderCache /* Saves those that arrived out of order to be inserted later at the correct pos */;
 	bool previewDirty = false;
 	int previewsPerPage = 20;
-	unsigned int processedPdfPreviews = 0;
-	unsigned int currentPreviewGeneration = 1;
 
 	void connectSignals();
 	void makePreviews(int page);
@@ -56,20 +51,20 @@ class MainWindow : public QMainWindow
 	void keyPressEvent(QKeyEvent *event) override;
 	void handleSearchResults(const QVector<SearchResult> &results);
 	void handleSearchError(QString error);
-	void createSearchResutlMenu(QMenu &menu, const QFileInfo &fileInfo);
+	void createSearchResultMenu(QMenu &menu, const QFileInfo &fileInfo);
 	void openDocument(QString path, int num);
 	void openFile(QString path);
 	void initSettingsTabs();
 	int currentSelectedScale();
 	void processShortcut(int key);
-	bool eventFilter(QObject *object, QEvent *event);
+	bool eventFilter(QObject *object, QEvent *event) override;
 
   private slots:
 	void lineEditReturnPressed();
 	void treeSearchItemActivated(QTreeWidgetItem *item, int i);
 	void showSearchResultsContextMenu(const QPoint &point);
 	void tabChanged();
-	void previewReceived(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration);
+	void previewReceived();
 	void comboScaleChanged(int i);
 	void spinPreviewPageValueChanged(int val);
 	void startIndexing();

gui/mainwindow.ui

@@ -18,16 +18,13 @@
     <item>
      <widget class="QLineEdit" name="txtSearch"/>
     </item>
-    <item>
-     <layout class="QHBoxLayout" name="horizontalLayout_3"/>
-    </item>
     <item>
      <widget class="QTabWidget" name="tabWidget">
       <property name="tabPosition">
        <enum>QTabWidget::South</enum>
       </property>
       <property name="currentIndex">
-       <number>1</number>
+       <number>2</number>
       </property>
       <widget class="QWidget" name="resultsTab">
        <attribute name="title">
@@ -82,7 +79,7 @@
              <x>0</x>
              <y>0</y>
              <width>1244</width>
-             <height>633</height>
+             <height>641</height>
             </rect>
            </property>
            <layout class="QHBoxLayout" name="horizontalLayout"/>
@@ -195,62 +192,6 @@
        </attribute>
        <layout class="QGridLayout" name="gridLayout">
         <item row="6" column="0">
-         <widget class="QLineEdit" name="txtIgnorePatterns"/>
-        </item>
-        <item row="11" column="0">
-         <widget class="QPushButton" name="btnStartIndexing">
-          <property name="text">
-           <string>Start indexing</string>
-          </property>
-         </widget>
-        </item>
-        <item row="1" column="0">
-         <widget class="QGroupBox" name="groupBoxPaths">
-          <property name="title">
-           <string>Add paths to scan</string>
-          </property>
-          <layout class="QGridLayout" name="gridLayout_2">
-           <item row="1" column="0">
-            <widget class="QLineEdit" name="txtPathScanAdd"/>
-           </item>
-           <item row="3" column="0" colspan="5">
-            <widget class="QListWidget" name="lstPaths"/>
-           </item>
-           <item row="1" column="3">
-            <widget class="QToolButton" name="btnDeletePath">
-             <property name="enabled">
-              <bool>false</bool>
-             </property>
-             <property name="text">
-              <string>Delete</string>
-             </property>
-            </widget>
-           </item>
-           <item row="1" column="1">
-            <widget class="QPushButton" name="btnChoosePath">
-             <property name="text">
-              <string>...</string>
-             </property>
-            </widget>
-           </item>
-           <item row="1" column="2">
-            <widget class="QPushButton" name="btnAddPath">
-             <property name="text">
-              <string>Add</string>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </item>
-        <item row="5" column="0">
-         <widget class="QLabel" name="label">
-          <property name="text">
-           <string>Ignore patterns, separated by ';'. Example: *.js;*Downloads*</string>
-          </property>
-         </widget>
-        </item>
-        <item row="9" column="0">
          <widget class="QGroupBox" name="groupBoxIndexProgress">
           <property name="contextMenuPolicy">
            <enum>Qt::PreventContextMenu</enum>
@@ -452,6 +393,108 @@
           </layout>
          </widget>
         </item>
+        <item row="2" column="0">
+         <widget class="QGroupBox" name="groupBoxIndexOptions">
+          <property name="title">
+           <string>Index options</string>
+          </property>
+          <layout class="QVBoxLayout" name="verticalLayout_11">
+           <item>
+            <widget class="QLabel" name="label">
+             <property name="text">
+              <string>Ignore patterns, separated by ';'. Example: *.js;*Downloads*:</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLineEdit" name="txtIgnorePatterns"/>
+           </item>
+           <item>
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QRadioButton" name="radioIndexEverything">
+             <property name="text">
+              <string>Index everything (metadata + file content)</string>
+             </property>
+             <property name="checked">
+              <bool>true</bool>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="chkFillContentForContentless">
+             <property name="enabled">
+              <bool>true</bool>
+             </property>
+             <property name="text">
+              <string>Index content for files previously indexed without content</string>
+             </property>
+             <property name="checked">
+              <bool>false</bool>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QRadioButton" name="radioMetadataOnly">
+             <property name="text">
+              <string>Index metadata only, don't process content of files</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </item>
+        <item row="8" column="0">
+         <widget class="QPushButton" name="btnStartIndexing">
+          <property name="text">
+           <string>Start indexing</string>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="0">
+         <widget class="QGroupBox" name="groupBoxPaths">
+          <property name="title">
+           <string>Add paths to scan</string>
+          </property>
+          <layout class="QGridLayout" name="gridLayout_2">
+           <item row="1" column="0">
+            <widget class="QLineEdit" name="txtPathScanAdd"/>
+           </item>
+           <item row="3" column="0" colspan="5">
+            <widget class="QListWidget" name="lstPaths"/>
+           </item>
+           <item row="1" column="3">
+            <widget class="QToolButton" name="btnDeletePath">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="text">
+              <string>Delete</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="1">
+            <widget class="QPushButton" name="btnChoosePath">
+             <property name="text">
+              <string>...</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="2">
+            <widget class="QPushButton" name="btnAddPath">
+             <property name="text">
+              <string>Add</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </item>
        </layout>
       </widget>
       <widget class="QWidget" name="settingsTab">
@@ -701,5 +744,22 @@
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources/>
- <connections/>
+ <connections>
+  <connection>
+   <sender>radioIndexEverything</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>chkFillContentForContentless</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>639</x>
+     <y>464</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>639</x>
+     <y>497</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
 </ui>

gui/previewcoordinator.cpp

@@ -0,0 +1,97 @@
+#include "previewcoordinator.h"
+#include <QFileInfo>
+
+PreviewCoordinator::PreviewCoordinator()
+{
+	this->ipcPreviewClient.moveToThread(&this->ipcClientThread);
+
+	connect(&ipcPreviewClient, &IPCPreviewClient::previewReceived, this, &PreviewCoordinator::handleReceivedPreview,
+			Qt::QueuedConnection);
+	connect(&ipcPreviewClient, &IPCPreviewClient::finished, this, [&] { emit completedGeneration(); });
+	connect(this, &PreviewCoordinator::ipcStartGeneration, &ipcPreviewClient, &IPCPreviewClient::startGeneration,
+			Qt::QueuedConnection);
+
+	this->ipcClientThread.start();
+}
+
+void PreviewCoordinator::init(const QVector<SearchResult> &searchResults)
+{
+	this->previewableSearchResults.clear();
+	for(const SearchResult &result : searchResults)
+	{
+		if(result.wasContentSearch)
+		{
+			QString path = result.fileData.absPath;
+			// HACK until we can preview them properly
+			if(path.endsWith(".html") || path.endsWith(".htm"))
+			{
+				continue;
+			}
+			QFileInfo info{path};
+			if(info.exists())
+			{
+				this->previewableSearchResults.append(result);
+			}
+		}
+	}
+}
+
+void PreviewCoordinator::setSocketPath(QString socketPath)
+{
+	this->socketPath = socketPath;
+	this->ipcPreviewClient.setSocketPath(socketPath);
+}
+
+int PreviewCoordinator::previewableCount() const
+{
+	return this->previewableSearchResults.count();
+}
+
+QSharedPointer<PreviewResult> PreviewCoordinator::resultAt(int index)
+{
+	if(this->previewResults.size() > index)
+	{
+		return {this->previewResults[index]};
+	}
+	return {nullptr};
+}
+
+const QVector<SearchResult> &PreviewCoordinator::getPreviewableSearchResults() const
+{
+	return this->previewableSearchResults;
+}
+
+void PreviewCoordinator::handleReceivedPreview(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration)
+{
+	if(previewGeneration < this->currentPreviewGeneration)
+	{
+		return;
+	}
+	if(!preview.isNull() && preview->hasPreview())
+	{
+		QString docPath = preview->getDocumentPath();
+		auto previewPage = preview->getPage();
+		int pos = previewOrder[docPath + QString::number(previewPage)];
+		this->previewResults[pos] = preview;
+		emit previewReady();
+	}
+}
+
+void PreviewCoordinator::startGeneration(RenderConfig config, const QVector<RenderTarget> &targets)
+{
+	++this->currentPreviewGeneration;
+
+	this->previewOrder.clear();
+	this->previewResults.clear();
+
+	this->previewResults.resize(targets.size());
+	this->previewResults.fill(nullptr);
+
+	int i = 0;
+	for(const RenderTarget &target : targets)
+	{
+		this->previewOrder[target.path + QString::number(target.page)] = i++;
+	}
+
+	emit ipcStartGeneration(config, targets);
+}

gui/previewcoordinator.h

@@ -0,0 +1,48 @@
+#ifndef PREVIEWCOORDINATOR_H
+#define PREVIEWCOORDINATOR_H
+#include <QVector>
+#include <QObject>
+#include <QThread>
+#include "searchresult.h"
+#include "previewresult.h"
+#include "ipcpreviewclient.h"
+#include "rendertarget.h"
+class PreviewCoordinator : public QObject
+{
+	Q_OBJECT
+  private:
+	QThread ipcClientThread;
+	IPCPreviewClient ipcPreviewClient;
+	QString socketPath;
+
+	QVector<QSharedPointer<PreviewResult>> previewResults;
+	QVector<SearchResult> previewableSearchResults;
+
+	unsigned int currentPreviewGeneration = 1;
+
+	/* Quick lookup table for the order a preview should have */
+	QHash<QString, int> previewOrder;
+
+  public:
+	PreviewCoordinator();
+
+	void init(const QVector<SearchResult> &searchResults);
+
+	int previewableCount() const;
+	const QVector<SearchResult> &getPreviewableSearchResults() const;
+
+	QSharedPointer<PreviewResult> resultAt(int index);
+
+	void setSocketPath(QString socketPath);
+  public slots:
+	void startGeneration(RenderConfig config, const QVector<RenderTarget> &targets);
+	void handleReceivedPreview(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration);
+
+  signals:
+	void previewReady();
+	void completedGeneration();
+	void error(QString);
+	void ipcStartGeneration(RenderConfig config, const QVector<RenderTarget> &targets);
+};
+
+#endif // PREVIEWCOORDINATOR_H

gui/previewgeneratorodt.cpp

@@ -24,7 +24,7 @@ QSharedPointer<PreviewResult> PreviewGeneratorOdt::generate(RenderConfig config,
 		throw LooqsGeneralException("Error while reading content.xml of " + documentPath);
 	}
 	TagStripperProcessor tsp;
-	QString content = tsp.process(entireContent).first().content;
+	QString content = tsp.process(entireContent).constFirst().content;
 
 	PreviewGeneratorPlainText plainTextGenerator;
 	result->setText(plainTextGenerator.generatePreviewText(content, config, info.fileName()));

gui/previewgeneratorpdf.cpp

@@ -20,6 +20,8 @@ Poppler::Document *PreviewGeneratorPdf::document(QString path)
 		return nullptr;
 	}
 	result->setRenderHint(Poppler::Document::TextAntialiasing);
+	result->setRenderHint(Poppler::Document::TextHinting);
+	result->setRenderHint(Poppler::Document::TextSlightHinting);
 
 	locker.relock();
 	documentcache.insert(path, result);

gui/previewgeneratorplaintext.cpp

@@ -195,7 +195,7 @@ QString PreviewGeneratorPlainText::generateLineBasedPreviewText(QTextStream &in,
 				  int totalWordsA = 0;
 				  int differentWordsB = 0;
 				  int totalWordsB = 0;
-				  for(int count : a.wordCountMap.values())
+				  for(int count : qAsConst(a.wordCountMap))
 				  {
 					  if(count > 0)
 					  {
@@ -203,7 +203,7 @@ QString PreviewGeneratorPlainText::generateLineBasedPreviewText(QTextStream &in,
 					  }
 					  totalWordsA += count;
 				  }
-				  for(int count : b.wordCountMap.values())
+				  for(int count : qAsConst(b.wordCountMap))
 				  {
 					  if(count > 0)
 					  {

shared/filesaver.cpp

@@ -38,18 +38,17 @@ SaveFileResult FileSaver::updateFile(QString path)
 	return saveFile(info);
 }
 
-int FileSaver::addFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
+int FileSaver::addFiles(const QVector<QString> paths)
 {
-	return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose);
+	return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1));
 }
 
-int FileSaver::updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
+int FileSaver::updateFiles(const QVector<QString> paths)
 {
-	return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose);
+	return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1));
 }
 
-int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
-							bool keepGoing, bool verbose)
+int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc)
 {
 	std::atomic<bool> terminate{false};
 	std::atomic<int> processedCount{0};
@@ -60,7 +59,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  {
 									  return;
 								  }
-								  if(verbose)
+								  if(this->fileSaverOptions.verbose)
 								  {
 									  Logger::info() << "Processing " << path << Qt::endl;
 								  }
@@ -68,7 +67,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  if(result == DBFAIL || result == PROCESSFAIL)
 								  {
 									  Logger::error() << "Failed to process " << path << Qt::endl;
-									  if(!keepGoing)
+									  if(!this->fileSaverOptions.keepGoing)
 									  {
 										  terminate = true;
 									  }
@@ -76,7 +75,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  else
 								  {
 									  ++processedCount;
-									  if(verbose)
+									  if(this->fileSaverOptions.verbose)
 									  {
 										  if(result == SKIPPED)
 										  {
@@ -120,11 +119,29 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 		{
 			if(canonicalPath.startsWith(excludedPath))
 			{
+				if(this->fileSaverOptions.verbose)
+				{
+					Logger::info() << "Skipped due to excluded path";
+				}
 				return SKIPPED;
 			}
 		}
 
-		if(fileInfo.size() > 0)
+		bool mustFillContent = this->fileSaverOptions.fillExistingContentless;
+		if(!mustFillContent)
+		{
+			mustFillContent = !this->fileSaverOptions.metadataOnly;
+			if(mustFillContent)
+			{
+				auto filetype = this->dbService->queryFileType(fileInfo.absolutePath());
+				if(filetype)
+				{
+					mustFillContent = filetype.value() == 'c';
+				}
+			}
+		}
+
+		if(fileInfo.size() > 0 && mustFillContent)
 		{
 			QProcess process;
 			QStringList args;
@@ -159,7 +176,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 			}
 		}
 	}
-	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData);
+	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.metadataOnly);
 	if(result == OK && processorReturnCode == OK_WASEMPTY)
 	{
 		return OK_WASEMPTY;

shared/filesaver.h

@@ -2,6 +2,7 @@
 #define FILESAVER_H
 #include <QSqlDatabase>
 #include <QFileInfo>
+#include "filesaveroptions.h"
 #include "pagedata.h"
 #include "filedata.h"
 #include "sqlitedbservice.h"
@@ -11,16 +12,21 @@ class FileSaver
   private:
 	SqliteDbService *dbService;
 	QStringList excludedPaths = Common::excludedPaths();
+	FileSaverOptions fileSaverOptions;
 
   public:
 	FileSaver(SqliteDbService &dbService);
 	SaveFileResult addFile(QString path);
 	SaveFileResult updateFile(QString path);
 	SaveFileResult saveFile(const QFileInfo &fileInfo);
-	int processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
-					 bool keepGoing, bool verbose);
-	int addFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
-	int updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
+	int processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc);
+	int addFiles(const QVector<QString> paths);
+	int updateFiles(const QVector<QString> paths);
+
+	void setFileSaverOptions(FileSaverOptions options)
+	{
+		this->fileSaverOptions = options;
+	}
 };
 
 #endif // FILESAVER_H

shared/filesaveroptions.h

@@ -0,0 +1,14 @@
+#ifndef FILESAVEROPTIONS_H
+#define FILESAVEROPTIONS_H
+
+class FileSaverOptions
+{
+  public:
+	bool verbose = false;
+	bool keepGoing = false;
+	bool metadataOnly = false;
+	/* Whether those previously explicitly without content should be filled */
+	bool fillExistingContentless = false;
+};
+
+#endif // FILESAVEROPTIONS_H

shared/filescanworker.cpp

@@ -12,6 +12,7 @@ FileScanWorker::FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &qu
 void FileScanWorker::run()
 {
 	FileSaver saver{*this->dbService};
+	saver.setFileSaverOptions(this->fileSaverOptions);
 	auto paths = queue->dequeue(batchsize);
 	for(QString &path : paths)
 	{
@@ -34,3 +35,8 @@ void FileScanWorker::run()
 	}
 	emit finished();
 }
+
+void FileScanWorker::setFileSaverOptions(FileSaverOptions options)
+{
+	this->fileSaverOptions = options;
+}

shared/filescanworker.h

@@ -15,12 +15,14 @@ class FileScanWorker : public QObject, public QRunnable
   protected:
 	SqliteDbService *dbService;
 	ConcurrentQueue<QString> *queue;
+	FileSaverOptions fileSaverOptions;
 	int batchsize;
 	std::atomic<bool> *stopToken;
 
   public:
 	FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &queue, int batchsize, std::atomic<bool> &stopToken);
 	void run() override;
+	void setFileSaverOptions(FileSaverOptions options);
   signals:
 	void result(FileScanResult);
 	void finished();

shared/indexer.cpp

@@ -73,16 +73,6 @@ void Indexer::setTargetPaths(QVector<QString> pathsToScan)
 	this->pathsToScan = pathsToScan;
 }
 
-void Indexer::setVerbose(bool verbose)
-{
-	this->verbose = verbose;
-}
-
-void Indexer::setKeepGoing(bool keepGoing)
-{
-	this->keepGoing = keepGoing;
-}
-
 void Indexer::requestCancellation()
 {
 	this->dirScanner->cancel();
@@ -108,6 +98,7 @@ void Indexer::launchWorker(ConcurrentQueue<QString> &queue, int batchsize)
 	FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize, this->workerCancellationToken);
 	connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult);
 	connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker);
+	runnable->setFileSaverOptions(this->fileSaverOptions);
 	++this->runningWorkers;
 	QThreadPool::globalInstance()->start(runnable);
 }
@@ -123,7 +114,7 @@ void Indexer::processFileScanResult(FileScanResult result)
 	if(isErrorSaveFileResult(result.second))
 	{
 		this->currentIndexResult.results.append(result);
-		if(!keepGoing)
+		if(!this->fileSaverOptions.keepGoing)
 		{
 			this->requestCancellation();
 			emit finished();
@@ -132,7 +123,7 @@ void Indexer::processFileScanResult(FileScanResult result)
 	}
 	else
 	{
-		if(verbose)
+		if(this->fileSaverOptions.verbose)
 		{
 			this->currentIndexResult.results.append(result);
 		}
@@ -175,3 +166,8 @@ void Indexer::processFinishedWorker()
 		emit finished();
 	}
 }
+
+void Indexer::setFileSaverOptions(FileSaverOptions options)
+{
+	this->fileSaverOptions = options;
+}

shared/indexer.h

@@ -52,8 +52,7 @@ class Indexer : public QObject
 {
 	Q_OBJECT
   protected:
-	bool verbose = false;
-	bool keepGoing = true;
+	FileSaverOptions fileSaverOptions;
 	SqliteDbService *db;
 
 	int progressReportThreshold = 50;
@@ -80,8 +79,8 @@ class Indexer : public QObject
 	void beginIndexing();
 	void setIgnorePattern(QStringList ignorePattern);
 	void setTargetPaths(QVector<QString> pathsToScan);
-	void setVerbose(bool verbose);
-	void setKeepGoing(bool keepGoing);
+
+	void setFileSaverOptions(FileSaverOptions options);
 
 	void requestCancellation();
 

shared/indexsyncer.cpp

@@ -7,21 +7,16 @@ IndexSyncer::IndexSyncer(SqliteDbService &dbService)
 	this->dbService = &dbService;
 }
 
+void IndexSyncer::setFileSaverOptions(FileSaverOptions options)
+{
+	fileSaverOptions = options;
+}
+
 void IndexSyncer::setDryRun(bool dryRun)
 {
 	this->dryRun = dryRun;
 }
 
-void IndexSyncer::setVerbose(bool verbose)
-{
-	this->verbose = verbose;
-}
-
-void IndexSyncer::setKeepGoing(bool keepGoing)
-{
-	this->keepGoing = keepGoing;
-}
-
 void IndexSyncer::setRemoveDeletedFromIndex(bool removeDeletedFromIndex)
 {
 	this->removeDeletedFromIndex = removeDeletedFromIndex;
@@ -35,7 +30,7 @@ void IndexSyncer::setPattern(QString pattern)
 void IndexSyncer::sync()
 {
 	this->stopToken.store(false, std::memory_order_relaxed);
-	FileSaver saver(*this->dbService);
+
 	QVector<FileData> files;
 	int offset = 0;
 	int limit = 10000;
@@ -87,7 +82,7 @@ void IndexSyncer::sync()
 						if(!this->dbService->deleteFile(fileData.absPath))
 						{
 							emit error("Error: Failed to delete " + fileData.absPath + " from the index");
-							if(!this->keepGoing)
+							if(!this->fileSaverOptions.keepGoing)
 							{
 								emit finished(totalUpdatesFilesCount, totalDeletedFilesCount, totalErroredFilesCount);
 								return;
@@ -104,13 +99,15 @@ void IndexSyncer::sync()
 			}
 		}
 
-		unsigned int updatedFilesCount = saver.updateFiles(filePathsToUpdate, keepGoing, verbose);
+		FileSaver saver(*this->dbService);
+		saver.setFileSaverOptions(this->fileSaverOptions);
+		unsigned int updatedFilesCount = saver.updateFiles(filePathsToUpdate);
 		unsigned int shouldHaveUpdatedCount = static_cast<unsigned int>(filePathsToUpdate.size());
 		if(updatedFilesCount != shouldHaveUpdatedCount)
 		{
 
 			totalErroredFilesCount += (shouldHaveUpdatedCount - updatedFilesCount);
-			if(!keepGoing)
+			if(!this->fileSaverOptions.keepGoing)
 			{
 				QString errorMsg = QString("Failed to update all files selected for updating in this batch. Updated") +
 								   updatedFilesCount + "out of" + shouldHaveUpdatedCount + "selected for updating";

shared/indexsyncer.h

@@ -1,16 +1,15 @@
 #ifndef INDEXSYNCER_H
 #define INDEXSYNCER_H
 #include "sqlitedbservice.h"
-
+#include "filesaveroptions.h"
 class IndexSyncer : public QObject
 {
 	Q_OBJECT
   private:
 	SqliteDbService *dbService = nullptr;
-	bool keepGoing = true;
+	FileSaverOptions fileSaverOptions;
 	bool removeDeletedFromIndex = true;
 	bool dryRun = false;
-	bool verbose = false;
 	QString pattern;
 
 	std::atomic<bool> stopToken{false};
@@ -18,12 +17,12 @@ class IndexSyncer : public QObject
   public:
 	IndexSyncer(SqliteDbService &dbService);
 
+	void setFileSaverOptions(FileSaverOptions options);
+
   public slots:
 	void sync();
 	void cancel();
 	void setDryRun(bool dryRun);
-	void setVerbose(bool verbose);
-	void setKeepGoing(bool keepGoing);
 	void setRemoveDeletedFromIndex(bool removeDeletedFromIndex);
 	void setPattern(QString pattern);
 

shared/looqsquery.cpp

@@ -7,6 +7,7 @@
 #include <optional>
 #include <algorithm>
 #include "looqsquery.h"
+#include "looqsgeneralexception.h"
 
 const QVector<Token> &LooqsQuery::getTokens() const
 {
@@ -180,8 +181,9 @@ LooqsQuery LooqsQuery::build(QString expression, TokenType loneWordsTokenType, b
 
 	QStringList loneWords;
 	LooqsQuery result;
-	QRegularExpression rx("((?<filtername>(\\.|\\w)+):(?<args>\\((?<innerargs>[^\\)]+)\\)|([^\\s])+)|(?<boolean>AND|OR)"
-						  "|(?<negation>!)|(?<bracket>\\(|\\))|(?<loneword>[^\\s]+))");
+	static QRegularExpression rx(
+		"((?<filtername>(\\.|\\w)+):(?<args>\\((?<innerargs>[^\\)]+)\\)|([^\\s])+)|(?<boolean>AND|OR)"
+		"|(?<negation>!)|(?<bracket>\\(|\\))|(?<loneword>[^\\s]+))");
 	QRegularExpressionMatchIterator i = rx.globalMatch(expression);
 	auto previousWasBool = [&result] { return !result.tokens.empty() && ((result.tokens.last().type & BOOL) == BOOL); };
 	auto previousWas = [&result](TokenType t) { return !result.tokens.empty() && (result.tokens.last().type == t); };

shared/looqsquery.h

@@ -2,7 +2,6 @@
 #define LOOQSQUERY_H
 #include <QString>
 #include <QVector>
-#include "looqsgeneralexception.h"
 #include "token.h"
 /* Fields that can be queried or sorted */
 enum QueryField
@@ -46,7 +45,7 @@ class LooqsQuery
 	void addToken(Token t);
 	void updateTokensMask()
 	{
-		for(const Token &t : tokens)
+		for(const Token &t : qAsConst(tokens))
 		{
 			this->tokensMask |= t.type;
 		}
@@ -92,14 +91,6 @@ class LooqsQuery
 		this->sortConditions = sortConditions;
 		updateTokensMask();
 	}
-
-	LooqsQuery(const LooqsQuery &o)
-	{
-		this->tokens = o.tokens;
-		this->sortConditions = o.sortConditions;
-		this->tokensMask = o.tokensMask;
-		this->limit = o.limit;
-	}
 };
 
 #endif // LOOQSQUERY_H

shared/nothingprocessor.h

@@ -10,7 +10,7 @@ class NothingProcessor : public Processor
 	NothingProcessor();
 
   public:
-	QVector<PageData> process(const QByteArray &data) const override
+	QVector<PageData> process(const QByteArray & /*data*/) const override
 	{
 		return {};
 	}

shared/odtprocessor.cpp

@@ -3,7 +3,7 @@
 #include "odtprocessor.h"
 #include "tagstripperprocessor.h"
 
-QVector<PageData> OdtProcessor::process(const QByteArray &data) const
+QVector<PageData> OdtProcessor::process(const QByteArray & /*data*/) const
 {
 	throw LooqsGeneralException("Not implemented yet");
 }

shared/paralleldirscanner.cpp

@@ -1,5 +1,3 @@
-#include "paralleldirscanner.h"
-
 #include <QRunnable>
 #include <QMutex>
 #include <QDirIterator>
@@ -7,7 +5,7 @@
 #include <QThreadPool>
 #include <functional>
 #include "dirscanworker.h"
-#include "logger.h"
+#include "paralleldirscanner.h"
 
 ParallelDirScanner::ParallelDirScanner()
 {

shared/shared.pro

@@ -74,6 +74,7 @@ HEADERS += sqlitesearch.h \
     encodingdetector.h \
     filedata.h \
     filesaver.h \
+    filesaveroptions.h \
     filescanworker.h \
     indexer.h \
     indexsyncer.h \

shared/sqlitedbservice.cpp

@@ -5,22 +5,6 @@
 #include "sqlitedbservice.h"
 #include "filedata.h"
 #include "logger.h"
-bool SqliteDbService::fileExistsInDatabase(QString path, qint64 mtime)
-{
-	auto query = QSqlQuery(dbFactory->forCurrentThread());
-	query.prepare("SELECT 1 FROM file WHERE path = ? and mtime = ?");
-	query.addBindValue(path);
-	query.addBindValue(mtime);
-	if(!query.exec())
-	{
-		throw LooqsGeneralException("Error while trying to query for file existance: " + query.lastError().text());
-	}
-	if(!query.next())
-	{
-		return false;
-	}
-	return query.value(0).toBool();
-}
 
 QVector<SearchResult> SqliteDbService::search(const LooqsQuery &query)
 {
@@ -29,20 +13,29 @@ QVector<SearchResult> SqliteDbService::search(const LooqsQuery &query)
 	return searcher.search(query);
 }
 
-bool SqliteDbService::fileExistsInDatabase(QString path)
+std::optional<QChar> SqliteDbService::queryFileType(QString absPath)
 {
-	auto query = QSqlQuery(dbFactory->forCurrentThread());
-	query.prepare("SELECT 1 FROM file WHERE path = ?");
-	query.addBindValue(path);
-	if(!query.exec())
-	{
-		throw LooqsGeneralException("Error while trying to query for file existance: " + query.lastError().text());
-	}
+	auto query = exec("SELCET filetype FROM file WHERE path = ?", {absPath});
 	if(!query.next())
 	{
-		return false;
+		return {};
 	}
-	return query.value(0).toBool();
+	return query.value(0).toChar();
+}
+
+bool SqliteDbService::fileExistsInDatabase(QString path)
+{
+	return execBool("SELECT 1 FROM file WHERE path = ?", {path});
+}
+
+bool SqliteDbService::fileExistsInDatabase(QString path, qint64 mtime)
+{
+	return execBool("SELECT 1 FROM file WHERE path = ? AND mtime = ?", {path, mtime});
+}
+
+bool SqliteDbService::fileExistsInDatabase(QString path, qint64 mtime, QChar fileType)
+{
+	return execBool("SELECT 1 FROM file WHERE path = ? AND mtime = ? AND filetype = ?", {path, mtime, fileType});
 }
 
 SqliteDbService::SqliteDbService(DatabaseFactory &dbFactory)
@@ -148,11 +141,40 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid
 	return true;
 }
 
-SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData)
+QSqlQuery SqliteDbService::exec(QString querystr, std::initializer_list<QVariant> args)
+{
+	auto query = QSqlQuery(dbFactory->forCurrentThread());
+	query.prepare(querystr);
+	for(const QVariant &v : args)
+	{
+		query.addBindValue(v);
+	}
+	if(!query.exec())
+	{
+		throw LooqsGeneralException("Error while trying to query for file existance: " + query.lastError().text());
+	}
+	return query;
+}
+
+bool SqliteDbService::execBool(QString querystr, std::initializer_list<QVariant> args)
+{
+	auto query = exec(querystr, args);
+	if(!query.next())
+	{
+		return false;
+	}
+	return query.value(0).toBool();
+}
+
+SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly)
 {
 	QString absPath = fileInfo.absoluteFilePath();
 	auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
-	QChar fileType = fileInfo.isDir() ? 'd' : 'f';
+	QChar fileType = fileInfo.isDir() ? 'd' : 'c';
+	if(pathsOnly)
+	{
+		fileType = 'f';
+	}
 
 	QSqlDatabase db = dbFactory->forCurrentThread();
 	QSqlQuery delQuery(db);
@@ -186,19 +208,23 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &
 		return DBFAIL;
 	}
 
-	int lastid = inserterQuery.lastInsertId().toInt();
-	if(!insertToFTS(false, db, lastid, pageData))
+	if(!pathsOnly)
 	{
-		db.rollback();
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
-		return DBFAIL;
-	}
-	if(!insertToFTS(true, db, lastid, pageData))
-	{
-		db.rollback();
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
-		return DBFAIL;
+		int lastid = inserterQuery.lastInsertId().toInt();
+		if(!insertToFTS(false, db, lastid, pageData))
+		{
+			db.rollback();
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+			return DBFAIL;
+		}
+		if(!insertToFTS(true, db, lastid, pageData))
+		{
+			db.rollback();
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+			return DBFAIL;
+		}
 	}
+
 	if(!db.commit())
 	{
 		db.rollback();

shared/sqlitedbservice.h

@@ -1,6 +1,8 @@
 #ifndef SQLITEDBSERVICE_H
 #define SQLITEDBSERVICE_H
 #include <QFileInfo>
+#include <optional>
+
 #include "databasefactory.h"
 #include "utils.h"
 #include "pagedata.h"
@@ -15,14 +17,20 @@ class SqliteDbService
 	DatabaseFactory *dbFactory = nullptr;
 	bool insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid, QVector<PageData> &pageData);
 
+	QSqlQuery exec(QString query, std::initializer_list<QVariant> args);
+	bool execBool(QString querystr, std::initializer_list<QVariant> args);
+
   public:
 	SqliteDbService(DatabaseFactory &dbFactory);
-	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData);
+	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly);
 	unsigned int getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit);
 	bool deleteFile(QString path);
 	bool fileExistsInDatabase(QString path);
 	bool fileExistsInDatabase(QString path, qint64 mtime);
+	bool fileExistsInDatabase(QString path, qint64 mtime, QChar filetype);
 	QVector<SearchResult> search(const LooqsQuery &query);
+
+	std::optional<QChar> queryFileType(QString absPath);
 };
 
 #endif // SQLITEDBSERVICE_H

shared/sqlitesearch.cpp

@@ -69,7 +69,7 @@ QString SqliteSearch::createSortSql(const QVector<SortCondition> sortConditions)
 QString SqliteSearch::escapeFtsArgument(QString ftsArg)
 {
 	QString result;
-	QRegularExpression extractor(R"#("([^"]*)"|([^\s]+))#");
+	static QRegularExpression extractor(R"#("([^"]*)"|([^\s]+))#");
 	QRegularExpressionMatchIterator i = extractor.globalMatch(ftsArg);
 	while(i.hasNext())
 	{
@@ -149,7 +149,6 @@ QPair<QString, QVector<QString>> SqliteSearch::createSql(const Token &token)
 QSqlQuery SqliteSearch::makeSqlQuery(const LooqsQuery &query)
 {
 	QString whereSql;
-	QString joinSql;
 	QVector<QString> bindValues;
 	bool isContentSearch = (query.getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT;
 	if(query.getTokens().isEmpty())