gui: Add index options group in index tab

Apparently poppler or something needs open() with write
flags to render pdfs with proper fonts.

Landlock guards file system write access, so this is fine.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2018-2022: Albert Schwarzkopf <looqs at quitesimple period org>
+Copyright (c) 2018-2023: Albert Schwarzkopf <looqs at quitesimple period org>
 
 looqs is made available under the following license: 
 

cli/commandadd.cpp

@@ -44,20 +44,30 @@ int CommandAdd::handle(QStringList arguments)
 						"Continue adding files, don't exit on first error. If this option is not given, looqs will "
 						"exit asap, but it's possible that a few files will still be processed. "
 						"Set -t 1 to avoid this behavior, but processing will be slower. "},
+					   {{"n", "no-content"}, "Only add paths to database. Do not index content"},
+					   {{"f", "fill-content"}, "Index content for files previously indexed with -n"},
+					   {"tags", "Comma-separated list of tags to assign"},
 					   {{"t", "threads"}, "Number of threads to use.", "threads"}});
-
 	parser.addHelpOption();
 	parser.addPositionalArgument("add", "Add paths to the index",
 								 "add [paths...]. If no path is given, read from stdin, one path per line.");
 
 	parser.process(arguments);
 	this->keepGoing = parser.isSet("continue");
+	bool pathsOnly = parser.isSet("no-content");
+	bool fillContent = parser.isSet("fill-content");
 	if(parser.isSet("threads"))
 	{
 		QString threadsCount = parser.value("threads");
 		QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
 	}
 
+	if(pathsOnly && fillContent)
+	{
+		Logger::error() << "Invalid options: -n and -f cannot both be set";
+		return EXIT_FAILURE;
+	}
+
 	QStringList files = parser.positionalArguments();
 
 	if(files.length() == 0)
@@ -71,9 +81,16 @@ int CommandAdd::handle(QStringList arguments)
 		}
 	}
 
+	FileSaverOptions fileSaverOptions;
+	fileSaverOptions.keepGoing = keepGoing;
+	fileSaverOptions.fillExistingContentless = fillContent;
+	fileSaverOptions.metadataOnly = pathsOnly;
+	fileSaverOptions.verbose = false;
+
 	indexer = new Indexer(*this->dbService);
+	indexer->setFileSaverOptions(fileSaverOptions);
+
 	indexer->setTargetPaths(files.toVector());
-	indexer->setKeepGoing(keepGoing);
 
 	connect(indexer, &Indexer::pathsCountChanged, this,
 			[](int pathsCount) { Logger::info() << "Found paths: " << pathsCount << Qt::endl; });

cli/commandupdate.cpp

@@ -40,8 +40,12 @@ int CommandUpdate::handle(QStringList arguments)
 
 	bool hasErrors = false;
 	IndexSyncer *syncer = new IndexSyncer(*this->dbService);
-	syncer->setKeepGoing(keepGoing);
+
-	syncer->setVerbose(verbose);
+	FileSaverOptions fileOptions;
+	fileOptions.keepGoing = keepGoing;
+	fileOptions.verbose = verbose;
+
+	syncer->setFileSaverOptions(fileOptions);
 	syncer->setPattern(pattern);
 	syncer->setDryRun(dryRun);
 	syncer->setRemoveDeletedFromIndex(deleteMissing);

gui/gui.pro

@@ -34,6 +34,7 @@ SOURCES += \
         main.cpp \
         mainwindow.cpp \
       clicklabel.cpp \
+    previewcoordinator.cpp \
     previewgenerator.cpp \
     previewgeneratormapfunctor.cpp \
     previewgeneratorodt.cpp \
@@ -54,6 +55,7 @@ HEADERS += \
     ipcserver.h \
         mainwindow.h \
     clicklabel.h \
+    previewcoordinator.h \
     previewgenerator.h \
     previewgeneratormapfunctor.h \
     previewgeneratorodt.h \

gui/main.cpp

@@ -28,7 +28,7 @@ void enableIpcSandbox()
 	policy->namespace_options = EXILE_UNSHARE_USER | EXILE_UNSHARE_MOUNT | EXILE_UNSHARE_NETWORK;
 	policy->no_new_privs = 1;
 	policy->drop_caps = 1;
-	policy->vow_promises = exile_vows_from_str("thread cpath rpath unix stdio proc error");
+	policy->vow_promises = exile_vows_from_str("thread cpath rpath wpath unix stdio proc error");
 	policy->mount_path_policies_to_chroot = 1;
 
 	QString ipcSocketPath = Common::ipcSocketPath();

gui/mainwindow.cpp

@@ -22,7 +22,6 @@
 #include "../shared/sqlitesearch.h"
 #include "../shared/looqsgeneralexception.h"
 #include "../shared/common.h"
-#include "ipcpreviewclient.h"
 #include "previewgenerator.h"
 #include "aboutdialog.h"
 
@@ -32,8 +31,7 @@ MainWindow::MainWindow(QWidget *parent, QString socketPath)
 	this->progressDialog.cancel(); // because constructing it shows it, quite weird
 	ui->setupUi(this);
 	setWindowTitle(QCoreApplication::applicationName());
-	this->ipcPreviewClient.moveToThread(&this->ipcClientThread);
+
-	this->ipcPreviewClient.setSocketPath(socketPath);
 	QSettings settings;
 
 	this->dbFactory = new DatabaseFactory(Common::databasePath());
@@ -78,7 +76,7 @@ MainWindow::MainWindow(QWidget *parent, QString socketPath)
 	ui->txtSearch->installEventFilter(this);
 	ui->scrollArea->viewport()->installEventFilter(this);
 
-	this->ipcClientThread.start();
+	this->previewCoordinator.setSocketPath(socketPath);
 }
 
 void MainWindow::addPathToIndex()
@@ -208,9 +206,9 @@ void MainWindow::connectSignals()
 			}
 		},
 		Qt::QueuedConnection);
-	connect(&ipcPreviewClient, &IPCPreviewClient::previewReceived, this, &MainWindow::previewReceived,
+	connect(&previewCoordinator, &PreviewCoordinator::previewReady, this, &MainWindow::previewReceived,
 			Qt::QueuedConnection);
-	connect(&ipcPreviewClient, &IPCPreviewClient::finished, this,
+	connect(&previewCoordinator, &PreviewCoordinator::completedGeneration, this,
 			[&]
 			{
 				this->ui->previewProcessBar->setValue(this->ui->previewProcessBar->maximum());
@@ -218,22 +216,24 @@ void MainWindow::connectSignals()
 				this->ui->comboPreviewFiles->setEnabled(true);
 				ui->txtSearch->setEnabled(true);
 			});
-	connect(&ipcPreviewClient, &IPCPreviewClient::error, this,
+	connect(&previewCoordinator, &PreviewCoordinator::error, this,
 			[this](QString msg)
 			{
 				qCritical() << msg << Qt::endl;
 				QMessageBox::critical(this, "IPC error", msg);
 			});
-
+	connect(ui->radioMetadataOnly, &QRadioButton::toggled, this,
-	connect(this, &MainWindow::startIpcPreviews, &ipcPreviewClient, &IPCPreviewClient::startGeneration,
+			[this](bool toggled)
-			Qt::QueuedConnection);
+			{
-	connect(this, &MainWindow::stopIpcPreviews, &ipcPreviewClient, &IPCPreviewClient::stopGeneration,
+				if(toggled)
-			Qt::QueuedConnection);
+				{
+					this->ui->chkFillContentForContentless->setChecked(false);
+				};
+			});
 }
 
 void MainWindow::exportFailedPaths()
 {
-
 	QString filename =
 		QString("/tmp/looqs_indexresult_failed_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_hhmmss"));
 	QFile outFile(filename);
@@ -266,8 +266,11 @@ void MainWindow::startIndexSync()
 	progressDialog.setValue(0);
 	progressDialog.open();
 
-	indexSyncer->setKeepGoing(true);
+	FileSaverOptions options;
-	indexSyncer->setVerbose(false);
+	options.keepGoing = true;
+	options.verbose = false;
+
+	indexSyncer->setFileSaverOptions(options);
 	indexSyncer->setDryRun(false);
 	indexSyncer->setRemoveDeletedFromIndex(true);
 
@@ -311,6 +314,15 @@ void MainWindow::startIndexing()
 	this->indexer->setTargetPaths(paths);
 	QString ignorePatterns = ui->txtIgnorePatterns->text();
 	this->indexer->setIgnorePattern(ignorePatterns.split(";"));
+
+	FileSaverOptions options;
+	options.fillExistingContentless =
+		ui->chkFillContentForContentless->isEnabled() && ui->chkFillContentForContentless->isChecked();
+	options.metadataOnly = ui->radioMetadataOnly->isChecked();
+	options.verbose = false;
+	options.keepGoing = true;
+
+	this->indexer->setFileSaverOptions(options);
 	this->indexer->beginIndexing();
 	QSettings settings;
 	settings.setValue("indexPaths", pathSettingsValue);
@@ -632,13 +644,17 @@ void MainWindow::saveSettings()
 	qApp->quit();
 }
 
-void MainWindow::previewReceived(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration)
+void MainWindow::previewReceived()
 {
-	if(previewGeneration < this->currentPreviewGeneration)
-	{
-		return;
-	}
 	this->ui->previewProcessBar->setValue(this->ui->previewProcessBar->value() + 1);
+	QBoxLayout *layout = static_cast<QBoxLayout *>(ui->scrollAreaWidgetContents->layout());
+	int index = layout->count();
+	if(index > 0)
+	{
+		--index;
+	}
+	QSharedPointer<PreviewResult> preview = this->previewCoordinator.resultAt(index);
+
 	if(!preview.isNull() && preview->hasPreview())
 	{
 		QString docPath = preview->getDocumentPath();
@@ -684,24 +700,7 @@ void MainWindow::previewReceived(QSharedPointer<PreviewResult> preview, unsigned
 
 		previewWidget->setLayout(previewLayout);
 
-		QBoxLayout *layout = static_cast<QBoxLayout *>(ui->scrollAreaWidgetContents->layout());
+		layout->insertWidget(index, previewWidget);
-		int pos = previewOrder[docPath + QString::number(previewPage)];
-		if(pos <= layout->count())
-		{
-			layout->insertWidget(pos, previewWidget);
-			for(auto it = previewWidgetOrderCache.constKeyValueBegin();
-				it != previewWidgetOrderCache.constKeyValueEnd(); it++)
-			{
-				if(it->first <= layout->count())
-				{
-					layout->insertWidget(it->first, it->second);
-				}
-			}
-		}
-		else
-		{
-			previewWidgetOrderCache[pos] = previewWidget;
-		}
 	}
 }
 
@@ -818,7 +817,6 @@ void MainWindow::lineEditReturnPressed()
 
 void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 {
-	this->previewableSearchResults.clear();
 	qDeleteAll(ui->scrollAreaWidgetContents->children());
 
 	ui->treeResultsList->clear();
@@ -827,6 +825,8 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 	ui->comboPreviewFiles->setVisible(true);
 	ui->lblTotalPreviewPagesCount->setText("");
 
+	this->previewCoordinator.init(results);
+
 	bool hasDeleted = false;
 	QHash<QString, bool> seenMap;
 	for(const SearchResult &result : results)
@@ -847,34 +847,29 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 			item->setText(3, this->locale().formattedDataSize(result.fileData.size));
 		}
 		bool exists = pathInfo.exists();
-		if(exists)
+		if(!exists)
-		{
-			if(result.wasContentSearch)
-			{
-				if(!pathInfo.suffix().contains("htm")) // hack until we can preview them properly...
-				{
-					if(PreviewGenerator::get(pathInfo) != nullptr)
-					{
-						this->previewableSearchResults.append(result);
-						if(!seenMap.contains(result.fileData.absPath))
-						{
-							ui->comboPreviewFiles->addItem(result.fileData.absPath);
-						}
-					}
-				}
-			}
-		}
-		else
 		{
 			hasDeleted = true;
 		}
 		seenMap[absPath] = true;
 	}
 
+	seenMap.clear();
+	for(const SearchResult &result : this->previewCoordinator.getPreviewableSearchResults())
+	{
+		const QString &absPath = result.fileData.absPath;
+		if(!seenMap.contains(absPath))
+		{
+			ui->comboPreviewFiles->addItem(absPath);
+		}
+		seenMap[absPath] = true;
+	}
+
 	ui->treeResultsList->resizeColumnToContents(0);
 	ui->treeResultsList->resizeColumnToContents(1);
 	ui->treeResultsList->resizeColumnToContents(2);
-	previewDirty = !this->previewableSearchResults.empty();
+
+	previewDirty = this->previewCoordinator.previewableCount() > 0;
 
 	ui->spinPreviewPage->setValue(1);
 
@@ -884,7 +879,7 @@ void MainWindow::handleSearchResults(const QVector<SearchResult> &results)
 	}
 
 	QString statusText = "Results: " + QString::number(results.size()) + " files";
-	statusText += ", previewable: " + QString::number(this->previewableSearchResults.count());
+	statusText += ", previewable: " + QString::number(this->previewCoordinator.previewableCount());
 	if(hasDeleted)
 	{
 		statusText += " WARNING: Some files are inaccessible. No preview available for those. Index may be out of sync";
@@ -901,7 +896,7 @@ int MainWindow::currentSelectedScale()
 
 void MainWindow::makePreviews(int page)
 {
-	if(this->previewableSearchResults.empty())
+	if(this->previewCoordinator.previewableCount() == 0)
 	{
 		return;
 	}
@@ -918,8 +913,7 @@ void MainWindow::makePreviews(int page)
 		ui->scrollAreaWidgetContents->setLayout(new QVBoxLayout());
 		ui->scrollAreaWidgetContents->layout()->setAlignment(Qt::AlignCenter);
 	}
-	ui->previewProcessBar->setMaximum(this->previewableSearchResults.size());
+	ui->previewProcessBar->setMaximum(this->previewCoordinator.previewableCount());
-	processedPdfPreviews = 0;
 
 	QVector<QString> wordsToHighlight;
 	QRegularExpression extractor(R"#("([^"]*)"|([^\s]+))#");
@@ -954,12 +948,9 @@ void MainWindow::makePreviews(int page)
 	renderConfig.scaleY = QGuiApplication::primaryScreen()->physicalDotsPerInchY() * (currentScale / 100.);
 	renderConfig.wordsToHighlight = wordsToHighlight;
 
-	this->previewOrder.clear();
-	this->previewWidgetOrderCache.clear();
-
 	int previewPos = 0;
 	QVector<RenderTarget> targets;
-	for(SearchResult &sr : this->previewableSearchResults)
+	for(const SearchResult &sr : this->previewCoordinator.getPreviewableSearchResults())
 	{
 		if(ui->comboPreviewFiles->currentIndex() != 0)
 		{
@@ -971,11 +962,8 @@ void MainWindow::makePreviews(int page)
 		RenderTarget renderTarget;
 		renderTarget.path = sr.fileData.absPath;
 		renderTarget.page = (int)sr.page;
-		targets.append(renderTarget);
 
-		int pos = previewPos - beginOffset;
+		targets.append(renderTarget);
-		this->previewOrder[renderTarget.path + QString::number(renderTarget.page)] = pos;
-		++previewPos;
 	}
 	int numpages = ceil(static_cast<double>(targets.size()) / previewsPerPage);
 	ui->spinPreviewPage->setMaximum(numpages);
@@ -985,12 +973,12 @@ void MainWindow::makePreviews(int page)
 	ui->previewProcessBar->setMaximum(targets.count());
 	ui->previewProcessBar->setMinimum(0);
 	ui->previewProcessBar->setValue(0);
-	ui->previewProcessBar->setVisible(this->previewableSearchResults.size() > 0);
+	ui->previewProcessBar->setVisible(this->previewCoordinator.previewableCount() > 0);
-	++this->currentPreviewGeneration;
 	this->ui->spinPreviewPage->setEnabled(false);
 	this->ui->comboPreviewFiles->setEnabled(false);
 	this->ui->txtSearch->setEnabled(false);
-	emit startIpcPreviews(renderConfig, targets);
+
+	this->previewCoordinator.startGeneration(renderConfig, targets);
 }
 
 void MainWindow::handleSearchError(QString error)
@@ -1006,11 +994,12 @@ void MainWindow::createSearchResultMenu(QMenu &menu, const QFileInfo &fileInfo)
 				   [&fileInfo] { QGuiApplication::clipboard()->setText(fileInfo.absoluteFilePath()); });
 	menu.addAction("Open containing folder", [this, &fileInfo] { this->openFile(fileInfo.absolutePath()); });
 
+	auto previewables = this->previewCoordinator.getPreviewableSearchResults();
 	auto result =
-		std::find_if(this->previewableSearchResults.begin(), this->previewableSearchResults.end(),
+		std::find_if(previewables.begin(), previewables.end(),
 					 [this, &fileInfo](SearchResult &a) { return fileInfo.absoluteFilePath() == a.fileData.absPath; });
 
-	if(result != this->previewableSearchResults.end())
+	if(result != previewables.end())
 	{
 		menu.addAction("Show previews for this file",
 					   [this, &fileInfo]
@@ -1069,7 +1058,6 @@ void MainWindow::showSearchResultsContextMenu(const QPoint &point)
 MainWindow::~MainWindow()
 {
 	syncerThread.terminate();
-	ipcClientThread.terminate();
 	delete this->indexSyncer;
 	delete this->dbService;
 	delete this->dbFactory;

gui/mainwindow.h

@@ -12,7 +12,7 @@
 #include <QProgressDialog>
 #include "../shared/looqsquery.h"
 #include "../shared/indexsyncer.h"
-#include "ipcpreviewclient.h"
+#include "previewcoordinator.h"
 #include "indexer.h"
 namespace Ui
 {
@@ -27,8 +27,9 @@ class MainWindow : public QMainWindow
 	DatabaseFactory *dbFactory;
 	SqliteDbService *dbService;
 	Ui::MainWindow *ui;
-	IPCPreviewClient ipcPreviewClient;
+
-	QThread ipcClientThread;
+	PreviewCoordinator previewCoordinator;
+
 	QThread syncerThread;
 	Indexer *indexer;
 	IndexSyncer *indexSyncer;
@@ -36,18 +37,12 @@ class MainWindow : public QMainWindow
 	QFileIconProvider iconProvider;
 	QSqlDatabase db;
 	QFutureWatcher<QVector<SearchResult>> searchWatcher;
-	QVector<SearchResult> previewableSearchResults;
 	LooqsQuery contentSearchQuery;
 	QVector<QString> searchHistory;
 	int currentSearchHistoryIndex = 0;
 	QString currentSavedSearchText;
-	QHash<QString, int> previewOrder; /* Quick lookup for the order a preview should have */
-	QMap<int, QWidget *>
-		previewWidgetOrderCache /* Saves those that arrived out of order to be inserted later at the correct pos */;
 	bool previewDirty = false;
 	int previewsPerPage = 20;
-	unsigned int processedPdfPreviews = 0;
-	unsigned int currentPreviewGeneration = 1;
 
 	void connectSignals();
 	void makePreviews(int page);
@@ -69,7 +64,7 @@ class MainWindow : public QMainWindow
 	void treeSearchItemActivated(QTreeWidgetItem *item, int i);
 	void showSearchResultsContextMenu(const QPoint &point);
 	void tabChanged();
-	void previewReceived(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration);
+	void previewReceived();
 	void comboScaleChanged(int i);
 	void spinPreviewPageValueChanged(int val);
 	void startIndexing();

gui/mainwindow.ui

@@ -18,16 +18,13 @@
     <item>
      <widget class="QLineEdit" name="txtSearch"/>
     </item>
-    <item>
-     <layout class="QHBoxLayout" name="horizontalLayout_3"/>
-    </item>
     <item>
      <widget class="QTabWidget" name="tabWidget">
       <property name="tabPosition">
        <enum>QTabWidget::South</enum>
       </property>
       <property name="currentIndex">
-       <number>1</number>
+       <number>2</number>
       </property>
       <widget class="QWidget" name="resultsTab">
        <attribute name="title">
@@ -82,7 +79,7 @@
              <x>0</x>
              <y>0</y>
              <width>1244</width>
-             <height>633</height>
+             <height>641</height>
             </rect>
            </property>
            <layout class="QHBoxLayout" name="horizontalLayout"/>
@@ -195,62 +192,6 @@
        </attribute>
        <layout class="QGridLayout" name="gridLayout">
         <item row="6" column="0">
-         <widget class="QLineEdit" name="txtIgnorePatterns"/>
-        </item>
-        <item row="11" column="0">
-         <widget class="QPushButton" name="btnStartIndexing">
-          <property name="text">
-           <string>Start indexing</string>
-          </property>
-         </widget>
-        </item>
-        <item row="1" column="0">
-         <widget class="QGroupBox" name="groupBoxPaths">
-          <property name="title">
-           <string>Add paths to scan</string>
-          </property>
-          <layout class="QGridLayout" name="gridLayout_2">
-           <item row="1" column="0">
-            <widget class="QLineEdit" name="txtPathScanAdd"/>
-           </item>
-           <item row="3" column="0" colspan="5">
-            <widget class="QListWidget" name="lstPaths"/>
-           </item>
-           <item row="1" column="3">
-            <widget class="QToolButton" name="btnDeletePath">
-             <property name="enabled">
-              <bool>false</bool>
-             </property>
-             <property name="text">
-              <string>Delete</string>
-             </property>
-            </widget>
-           </item>
-           <item row="1" column="1">
-            <widget class="QPushButton" name="btnChoosePath">
-             <property name="text">
-              <string>...</string>
-             </property>
-            </widget>
-           </item>
-           <item row="1" column="2">
-            <widget class="QPushButton" name="btnAddPath">
-             <property name="text">
-              <string>Add</string>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </item>
-        <item row="5" column="0">
-         <widget class="QLabel" name="label">
-          <property name="text">
-           <string>Ignore patterns, separated by ';'. Example: *.js;*Downloads*</string>
-          </property>
-         </widget>
-        </item>
-        <item row="9" column="0">
          <widget class="QGroupBox" name="groupBoxIndexProgress">
           <property name="contextMenuPolicy">
            <enum>Qt::PreventContextMenu</enum>
@@ -452,6 +393,102 @@
           </layout>
          </widget>
         </item>
+        <item row="2" column="0">
+         <widget class="QGroupBox" name="groupBoxIndexOptions">
+          <property name="title">
+           <string>Index options</string>
+          </property>
+          <layout class="QVBoxLayout" name="verticalLayout_11">
+           <item>
+            <widget class="QLabel" name="label">
+             <property name="text">
+              <string>Ignore patterns, separated by ';'. Example: *.js;*Downloads*:</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QLineEdit" name="txtIgnorePatterns"/>
+           </item>
+           <item>
+            <widget class="Line" name="line">
+             <property name="orientation">
+              <enum>Qt::Horizontal</enum>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QRadioButton" name="radioIndexEverything">
+             <property name="text">
+              <string>Index everything (metadata + file content)</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="chkFillContentForContentless">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="text">
+              <string>Index content for files previously indexed without content</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QRadioButton" name="radioMetadataOnly">
+             <property name="text">
+              <string>Index metadata only, don't process content of files</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </item>
+        <item row="8" column="0">
+         <widget class="QPushButton" name="btnStartIndexing">
+          <property name="text">
+           <string>Start indexing</string>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="0">
+         <widget class="QGroupBox" name="groupBoxPaths">
+          <property name="title">
+           <string>Add paths to scan</string>
+          </property>
+          <layout class="QGridLayout" name="gridLayout_2">
+           <item row="1" column="0">
+            <widget class="QLineEdit" name="txtPathScanAdd"/>
+           </item>
+           <item row="3" column="0" colspan="5">
+            <widget class="QListWidget" name="lstPaths"/>
+           </item>
+           <item row="1" column="3">
+            <widget class="QToolButton" name="btnDeletePath">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="text">
+              <string>Delete</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="1">
+            <widget class="QPushButton" name="btnChoosePath">
+             <property name="text">
+              <string>...</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="2">
+            <widget class="QPushButton" name="btnAddPath">
+             <property name="text">
+              <string>Add</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </widget>
+        </item>
        </layout>
       </widget>
       <widget class="QWidget" name="settingsTab">
@@ -701,5 +738,22 @@
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources/>
- <connections/>
+ <connections>
+  <connection>
+   <sender>radioIndexEverything</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>chkFillContentForContentless</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>639</x>
+     <y>464</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>639</x>
+     <y>497</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
 </ui>

gui/previewcoordinator.cpp

@@ -0,0 +1,97 @@
+#include "previewcoordinator.h"
+#include <QFileInfo>
+
+PreviewCoordinator::PreviewCoordinator()
+{
+	this->ipcPreviewClient.moveToThread(&this->ipcClientThread);
+
+	connect(&ipcPreviewClient, &IPCPreviewClient::previewReceived, this, &PreviewCoordinator::handleReceivedPreview,
+			Qt::QueuedConnection);
+	connect(&ipcPreviewClient, &IPCPreviewClient::finished, this, [&] { emit completedGeneration(); });
+	connect(this, &PreviewCoordinator::ipcStartGeneration, &ipcPreviewClient, &IPCPreviewClient::startGeneration,
+			Qt::QueuedConnection);
+
+	this->ipcClientThread.start();
+}
+
+void PreviewCoordinator::init(const QVector<SearchResult> &searchResults)
+{
+	this->previewableSearchResults.clear();
+	for(const SearchResult &result : searchResults)
+	{
+		if(result.wasContentSearch)
+		{
+			QString path = result.fileData.absPath;
+			// HACK until we can preview them properly
+			if(path.endsWith(".html") || path.endsWith(".htm"))
+			{
+				continue;
+			}
+			QFileInfo info{path};
+			if(info.exists())
+			{
+				this->previewableSearchResults.append(result);
+			}
+		}
+	}
+}
+
+void PreviewCoordinator::setSocketPath(QString socketPath)
+{
+	this->socketPath = socketPath;
+	this->ipcPreviewClient.setSocketPath(socketPath);
+}
+
+int PreviewCoordinator::previewableCount() const
+{
+	return this->previewableSearchResults.count();
+}
+
+QSharedPointer<PreviewResult> PreviewCoordinator::resultAt(int index)
+{
+	if(this->previewResults.size() > index)
+	{
+		return {this->previewResults[index]};
+	}
+	return {nullptr};
+}
+
+const QVector<SearchResult> &PreviewCoordinator::getPreviewableSearchResults() const
+{
+	return this->previewableSearchResults;
+}
+
+void PreviewCoordinator::handleReceivedPreview(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration)
+{
+	if(previewGeneration < this->currentPreviewGeneration)
+	{
+		return;
+	}
+	if(!preview.isNull() && preview->hasPreview())
+	{
+		QString docPath = preview->getDocumentPath();
+		auto previewPage = preview->getPage();
+		int pos = previewOrder[docPath + QString::number(previewPage)];
+		this->previewResults[pos] = preview;
+		emit previewReady();
+	}
+}
+
+void PreviewCoordinator::startGeneration(RenderConfig config, const QVector<RenderTarget> &targets)
+{
+	++this->currentPreviewGeneration;
+
+	this->previewOrder.clear();
+	this->previewResults.clear();
+
+	this->previewResults.resize(targets.size());
+	this->previewResults.fill(nullptr);
+
+	int i = 0;
+	for(const RenderTarget &target : targets)
+	{
+		this->previewOrder[target.path + QString::number(target.page)] = i++;
+	}
+
+	emit ipcStartGeneration(config, targets);
+}

gui/previewcoordinator.h

@@ -0,0 +1,48 @@
+#ifndef PREVIEWCOORDINATOR_H
+#define PREVIEWCOORDINATOR_H
+#include <QVector>
+#include <QObject>
+#include <QThread>
+#include "searchresult.h"
+#include "previewresult.h"
+#include "ipcpreviewclient.h"
+#include "rendertarget.h"
+class PreviewCoordinator : public QObject
+{
+	Q_OBJECT
+  private:
+	QThread ipcClientThread;
+	IPCPreviewClient ipcPreviewClient;
+	QString socketPath;
+
+	QVector<QSharedPointer<PreviewResult>> previewResults;
+	QVector<SearchResult> previewableSearchResults;
+
+	unsigned int currentPreviewGeneration = 1;
+
+	/* Quick lookup table for the order a preview should have */
+	QHash<QString, int> previewOrder;
+
+  public:
+	PreviewCoordinator();
+
+	void init(const QVector<SearchResult> &searchResults);
+
+	int previewableCount() const;
+	const QVector<SearchResult> &getPreviewableSearchResults() const;
+
+	QSharedPointer<PreviewResult> resultAt(int index);
+
+	void setSocketPath(QString socketPath);
+  public slots:
+	void startGeneration(RenderConfig config, const QVector<RenderTarget> &targets);
+	void handleReceivedPreview(QSharedPointer<PreviewResult> preview, unsigned int previewGeneration);
+
+  signals:
+	void previewReady();
+	void completedGeneration();
+	void error(QString);
+	void ipcStartGeneration(RenderConfig config, const QVector<RenderTarget> &targets);
+};
+
+#endif // PREVIEWCOORDINATOR_H

gui/previewgeneratorpdf.cpp

@@ -20,6 +20,8 @@ Poppler::Document *PreviewGeneratorPdf::document(QString path)
 		return nullptr;
 	}
 	result->setRenderHint(Poppler::Document::TextAntialiasing);
+	result->setRenderHint(Poppler::Document::TextHinting);
+	result->setRenderHint(Poppler::Document::TextSlightHinting);
 
 	locker.relock();
 	documentcache.insert(path, result);

shared/filesaver.cpp

@@ -38,18 +38,17 @@ SaveFileResult FileSaver::updateFile(QString path)
 	return saveFile(info);
 }
 
-int FileSaver::addFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
+int FileSaver::addFiles(const QVector<QString> paths)
 {
-	return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose);
+	return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1));
 }
 
-int FileSaver::updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
+int FileSaver::updateFiles(const QVector<QString> paths)
 {
-	return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose);
+	return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1));
 }
 
-int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
+int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc)
-							bool keepGoing, bool verbose)
 {
 	std::atomic<bool> terminate{false};
 	std::atomic<int> processedCount{0};
@@ -60,7 +59,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  {
 									  return;
 								  }
-								  if(verbose)
+								  if(this->fileSaverOptions.verbose)
 								  {
 									  Logger::info() << "Processing " << path << Qt::endl;
 								  }
@@ -68,7 +67,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  if(result == DBFAIL || result == PROCESSFAIL)
 								  {
 									  Logger::error() << "Failed to process " << path << Qt::endl;
-									  if(!keepGoing)
+									  if(!this->fileSaverOptions.keepGoing)
 									  {
 										  terminate = true;
 									  }
@@ -76,7 +75,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
 								  else
 								  {
 									  ++processedCount;
-									  if(verbose)
+									  if(this->fileSaverOptions.verbose)
 									  {
 										  if(result == SKIPPED)
 										  {
@@ -120,11 +119,29 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 		{
 			if(canonicalPath.startsWith(excludedPath))
 			{
+				if(this->fileSaverOptions.verbose)
+				{
+					Logger::info() << "Skipped due to excluded path";
+				}
 				return SKIPPED;
 			}
 		}
 
-		if(fileInfo.size() > 0)
+		bool mustFillContent = this->fileSaverOptions.fillExistingContentless;
+		if(!mustFillContent)
+		{
+			mustFillContent = !this->fileSaverOptions.metadataOnly;
+			if(mustFillContent)
+			{
+				auto filetype = this->dbService->queryFileType(fileInfo.absolutePath());
+				if(filetype)
+				{
+					mustFillContent = filetype.value() == 'c';
+				}
+			}
+		}
+
+		if(fileInfo.size() > 0 && mustFillContent)
 		{
 			QProcess process;
 			QStringList args;
@@ -159,7 +176,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
 			}
 		}
 	}
-	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData);
+	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.metadataOnly);
 	if(result == OK && processorReturnCode == OK_WASEMPTY)
 	{
 		return OK_WASEMPTY;

shared/filesaver.h

@@ -2,6 +2,7 @@
 #define FILESAVER_H
 #include <QSqlDatabase>
 #include <QFileInfo>
+#include "filesaveroptions.h"
 #include "pagedata.h"
 #include "filedata.h"
 #include "sqlitedbservice.h"
@@ -11,16 +12,21 @@ class FileSaver
   private:
 	SqliteDbService *dbService;
 	QStringList excludedPaths = Common::excludedPaths();
+	FileSaverOptions fileSaverOptions;
 
   public:
 	FileSaver(SqliteDbService &dbService);
 	SaveFileResult addFile(QString path);
 	SaveFileResult updateFile(QString path);
 	SaveFileResult saveFile(const QFileInfo &fileInfo);
-	int processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
+	int processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc);
-					 bool keepGoing, bool verbose);
+	int addFiles(const QVector<QString> paths);
-	int addFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
+	int updateFiles(const QVector<QString> paths);
-	int updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
+
+	void setFileSaverOptions(FileSaverOptions options)
+	{
+		this->fileSaverOptions = options;
+	}
 };
 
 #endif // FILESAVER_H

shared/filesaveroptions.h

@@ -0,0 +1,14 @@
+#ifndef FILESAVEROPTIONS_H
+#define FILESAVEROPTIONS_H
+
+class FileSaverOptions
+{
+  public:
+	bool verbose = false;
+	bool keepGoing = false;
+	bool metadataOnly = false;
+	/* Whether those previously explicitly without content should be filled */
+	bool fillExistingContentless = false;
+};
+
+#endif // FILESAVEROPTIONS_H

shared/filescanworker.cpp

@@ -12,6 +12,7 @@ FileScanWorker::FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &qu
 void FileScanWorker::run()
 {
 	FileSaver saver{*this->dbService};
+	saver.setFileSaverOptions(this->fileSaverOptions);
 	auto paths = queue->dequeue(batchsize);
 	for(QString &path : paths)
 	{
@@ -34,3 +35,8 @@ void FileScanWorker::run()
 	}
 	emit finished();
 }
+
+void FileScanWorker::setFileSaverOptions(FileSaverOptions options)
+{
+	this->fileSaverOptions = options;
+}

shared/filescanworker.h

@@ -15,12 +15,14 @@ class FileScanWorker : public QObject, public QRunnable
   protected:
 	SqliteDbService *dbService;
 	ConcurrentQueue<QString> *queue;
+	FileSaverOptions fileSaverOptions;
 	int batchsize;
 	std::atomic<bool> *stopToken;
 
   public:
 	FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &queue, int batchsize, std::atomic<bool> &stopToken);
 	void run() override;
+	void setFileSaverOptions(FileSaverOptions options);
   signals:
 	void result(FileScanResult);
 	void finished();

shared/indexer.cpp

@@ -73,16 +73,6 @@ void Indexer::setTargetPaths(QVector<QString> pathsToScan)
 	this->pathsToScan = pathsToScan;
 }
 
-void Indexer::setVerbose(bool verbose)
-{
-	this->verbose = verbose;
-}
-
-void Indexer::setKeepGoing(bool keepGoing)
-{
-	this->keepGoing = keepGoing;
-}
-
 void Indexer::requestCancellation()
 {
 	this->dirScanner->cancel();
@@ -108,6 +98,7 @@ void Indexer::launchWorker(ConcurrentQueue<QString> &queue, int batchsize)
 	FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize, this->workerCancellationToken);
 	connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult);
 	connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker);
+	runnable->setFileSaverOptions(this->fileSaverOptions);
 	++this->runningWorkers;
 	QThreadPool::globalInstance()->start(runnable);
 }
@@ -123,7 +114,7 @@ void Indexer::processFileScanResult(FileScanResult result)
 	if(isErrorSaveFileResult(result.second))
 	{
 		this->currentIndexResult.results.append(result);
-		if(!keepGoing)
+		if(!this->fileSaverOptions.keepGoing)
 		{
 			this->requestCancellation();
 			emit finished();
@@ -132,7 +123,7 @@ void Indexer::processFileScanResult(FileScanResult result)
 	}
 	else
 	{
-		if(verbose)
+		if(this->fileSaverOptions.verbose)
 		{
 			this->currentIndexResult.results.append(result);
 		}
@@ -175,3 +166,8 @@ void Indexer::processFinishedWorker()
 		emit finished();
 	}
 }
+
+void Indexer::setFileSaverOptions(FileSaverOptions options)
+{
+	this->fileSaverOptions = options;
+}

shared/indexer.h

@@ -52,8 +52,7 @@ class Indexer : public QObject
 {
 	Q_OBJECT
   protected:
-	bool verbose = false;
+	FileSaverOptions fileSaverOptions;
-	bool keepGoing = true;
 	SqliteDbService *db;
 
 	int progressReportThreshold = 50;
@@ -80,8 +79,8 @@ class Indexer : public QObject
 	void beginIndexing();
 	void setIgnorePattern(QStringList ignorePattern);
 	void setTargetPaths(QVector<QString> pathsToScan);
-	void setVerbose(bool verbose);
+
-	void setKeepGoing(bool keepGoing);
+	void setFileSaverOptions(FileSaverOptions options);
 
 	void requestCancellation();
 

shared/indexsyncer.cpp

@@ -7,21 +7,16 @@ IndexSyncer::IndexSyncer(SqliteDbService &dbService)
 	this->dbService = &dbService;
 }
 
+void IndexSyncer::setFileSaverOptions(FileSaverOptions options)
+{
+	fileSaverOptions = options;
+}
+
 void IndexSyncer::setDryRun(bool dryRun)
 {
 	this->dryRun = dryRun;
 }
 
-void IndexSyncer::setVerbose(bool verbose)
-{
-	this->verbose = verbose;
-}
-
-void IndexSyncer::setKeepGoing(bool keepGoing)
-{
-	this->keepGoing = keepGoing;
-}
-
 void IndexSyncer::setRemoveDeletedFromIndex(bool removeDeletedFromIndex)
 {
 	this->removeDeletedFromIndex = removeDeletedFromIndex;
@@ -35,7 +30,7 @@ void IndexSyncer::setPattern(QString pattern)
 void IndexSyncer::sync()
 {
 	this->stopToken.store(false, std::memory_order_relaxed);
-	FileSaver saver(*this->dbService);
+
 	QVector<FileData> files;
 	int offset = 0;
 	int limit = 10000;
@@ -87,7 +82,7 @@ void IndexSyncer::sync()
 						if(!this->dbService->deleteFile(fileData.absPath))
 						{
 							emit error("Error: Failed to delete " + fileData.absPath + " from the index");
-							if(!this->keepGoing)
+							if(!this->fileSaverOptions.keepGoing)
 							{
 								emit finished(totalUpdatesFilesCount, totalDeletedFilesCount, totalErroredFilesCount);
 								return;
@@ -104,13 +99,15 @@ void IndexSyncer::sync()
 			}
 		}
 
-		unsigned int updatedFilesCount = saver.updateFiles(filePathsToUpdate, keepGoing, verbose);
+		FileSaver saver(*this->dbService);
+		saver.setFileSaverOptions(this->fileSaverOptions);
+		unsigned int updatedFilesCount = saver.updateFiles(filePathsToUpdate);
 		unsigned int shouldHaveUpdatedCount = static_cast<unsigned int>(filePathsToUpdate.size());
 		if(updatedFilesCount != shouldHaveUpdatedCount)
 		{
 
 			totalErroredFilesCount += (shouldHaveUpdatedCount - updatedFilesCount);
-			if(!keepGoing)
+			if(!this->fileSaverOptions.keepGoing)
 			{
 				QString errorMsg = QString("Failed to update all files selected for updating in this batch. Updated") +
 								   updatedFilesCount + "out of" + shouldHaveUpdatedCount + "selected for updating";

shared/indexsyncer.h

@@ -1,16 +1,15 @@
 #ifndef INDEXSYNCER_H
 #define INDEXSYNCER_H
 #include "sqlitedbservice.h"
-
+#include "filesaveroptions.h"
 class IndexSyncer : public QObject
 {
 	Q_OBJECT
   private:
 	SqliteDbService *dbService = nullptr;
-	bool keepGoing = true;
+	FileSaverOptions fileSaverOptions;
 	bool removeDeletedFromIndex = true;
 	bool dryRun = false;
-	bool verbose = false;
 	QString pattern;
 
 	std::atomic<bool> stopToken{false};
@@ -18,12 +17,12 @@ class IndexSyncer : public QObject
   public:
 	IndexSyncer(SqliteDbService &dbService);
 
+	void setFileSaverOptions(FileSaverOptions options);
+
   public slots:
 	void sync();
 	void cancel();
 	void setDryRun(bool dryRun);
-	void setVerbose(bool verbose);
-	void setKeepGoing(bool keepGoing);
 	void setRemoveDeletedFromIndex(bool removeDeletedFromIndex);
 	void setPattern(QString pattern);
 

shared/shared.pro

@@ -74,6 +74,7 @@ HEADERS += sqlitesearch.h \
     encodingdetector.h \
     filedata.h \
     filesaver.h \
+    filesaveroptions.h \
     filescanworker.h \
     indexer.h \
     indexsyncer.h \

shared/sqlitedbservice.cpp

@@ -29,6 +29,22 @@ QVector<SearchResult> SqliteDbService::search(const LooqsQuery &query)
 	return searcher.search(query);
 }
 
+std::optional<QChar> SqliteDbService::queryFileType(QString absPath)
+{
+	auto query = QSqlQuery(dbFactory->forCurrentThread());
+	query.prepare("SELECT filetype FROM file WHERE path = ?");
+	query.addBindValue(absPath);
+	if(!query.exec())
+	{
+		throw LooqsGeneralException("Error while trying to query for file type: " + query.lastError().text());
+	}
+	if(!query.next())
+	{
+		return {};
+	}
+	return query.value(0).toChar();
+}
+
 bool SqliteDbService::fileExistsInDatabase(QString path)
 {
 	auto query = QSqlQuery(dbFactory->forCurrentThread());
@@ -148,11 +164,15 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid
 	return true;
 }
 
-SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData)
+SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly)
 {
 	QString absPath = fileInfo.absoluteFilePath();
 	auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
-	QChar fileType = fileInfo.isDir() ? 'd' : 'f';
+	QChar fileType = fileInfo.isDir() ? 'd' : 'c';
+	if(pathsOnly)
+	{
+		fileType = 'f';
+	}
 
 	QSqlDatabase db = dbFactory->forCurrentThread();
 	QSqlQuery delQuery(db);
@@ -186,19 +206,23 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &
 		return DBFAIL;
 	}
 
-	int lastid = inserterQuery.lastInsertId().toInt();
+	if(!pathsOnly)
-	if(!insertToFTS(false, db, lastid, pageData))
 	{
-		db.rollback();
+		int lastid = inserterQuery.lastInsertId().toInt();
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+		if(!insertToFTS(false, db, lastid, pageData))
-		return DBFAIL;
+		{
-	}
+			db.rollback();
-	if(!insertToFTS(true, db, lastid, pageData))
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
-	{
+			return DBFAIL;
-		db.rollback();
+		}
-		Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+		if(!insertToFTS(true, db, lastid, pageData))
-		return DBFAIL;
+		{
+			db.rollback();
+			Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
+			return DBFAIL;
+		}
 	}
+
 	if(!db.commit())
 	{
 		db.rollback();

shared/sqlitedbservice.h

@@ -1,6 +1,8 @@
 #ifndef SQLITEDBSERVICE_H
 #define SQLITEDBSERVICE_H
 #include <QFileInfo>
+#include <optional>
+
 #include "databasefactory.h"
 #include "utils.h"
 #include "pagedata.h"
@@ -17,12 +19,14 @@ class SqliteDbService
 
   public:
 	SqliteDbService(DatabaseFactory &dbFactory);
-	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData);
+	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly);
 	unsigned int getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit);
 	bool deleteFile(QString path);
 	bool fileExistsInDatabase(QString path);
 	bool fileExistsInDatabase(QString path, qint64 mtime);
 	QVector<SearchResult> search(const LooqsQuery &query);
+
+	std::optional<QChar> queryFileType(QString absPath);
 };
 
 #endif // SQLITEDBSERVICE_H