Begin a C++ cli and remove the Python scripts

cli/commandadd.cpp

@@ -0,0 +1,196 @@
+#include <QFileInfo>
+#include <QDebug>
+#include <QSqlQuery>
+#include <QSqlError>
+#include <QDateTime>
+#include <QMap>
+#include <QTextStream>
+#include <QException>
+#include <QCommandLineParser>
+#include <QMutex>
+#include <QMutexLocker>
+#include <QtConcurrent/QtConcurrentMap>
+#include "processor.h"
+#include "pdfprocessor.h"
+#include "commandadd.h"
+#include "defaulttextprocessor.h"
+#include "tagstripperprocessor.h"
+#include "nothingprocessor.h"
+#include "odtprocessor.h"
+#include "odsprocessor.h"
+#include "utils.h"
+static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
+static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
+static NothingProcessor *nothingProcessor = new NothingProcessor();
+static OdtProcessor *odtProcessor = new OdtProcessor();
+static OdsProcessor *odsProcessor = new OdsProcessor();
+
+static QMap<QString, Processor *> processors{
+	{"pdf", new PdfProcessor()},	{"txt", defaultTextProcessor}, {"md", defaultTextProcessor},
+	{"py", defaultTextProcessor},	{"xml", nothingProcessor},	   {"html", tagStripperProcessor},
+	{"java", defaultTextProcessor}, {"js", defaultTextProcessor},  {"cpp", defaultTextProcessor},
+	{"c", defaultTextProcessor},	{"sql", defaultTextProcessor}, {"odt", odtProcessor},
+	{"ods", odsProcessor}};
+
+AddFileResult CommandAdd::addFile(QString path)
+{
+	QFileInfo info(path);
+	QString absPath = info.absoluteFilePath();
+	auto mtime = info.lastModified().toSecsSinceEpoch();
+	QChar fileType = info.isDir() ? 'd' : 'f';
+
+	QSqlDatabase db = dbConnection();
+	if(fileExistsInDatabase(db, absPath, mtime))
+	{
+		return SKIPPED;
+	}
+
+	Processor *processor = processors.value(info.suffix(), nothingProcessor);
+	QVector<PageData> pageData;
+	if(processor->PREFERED_DATA_SOURCE == FILEPATH)
+	{
+		pageData = processor->process(absPath);
+	}
+	else
+	{
+		pageData = processor->process(Utils::readFile(absPath));
+	}
+
+	if(pageData.isEmpty())
+	{
+		qDebug() << "Could not get any content for " << absPath;
+	}
+
+	// Workaround to "database is locked" error. Perhaps try WAL mode etc.
+	// QMutexLocker locker(&writeMutex);
+	if(!db.transaction())
+	{
+		qDebug() << "Failed to open transaction for " << absPath << " : " << db.lastError();
+		return DBFAIL;
+	}
+
+	QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db);
+	delQuery.addBindValue(absPath);
+	if(!delQuery.exec())
+	{
+		qDebug() << "Failed DELETE query" << delQuery.lastError();
+		db.rollback();
+		return DBFAIL;
+	}
+
+	QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db);
+	inserterQuery.addBindValue(absPath);
+	inserterQuery.addBindValue(mtime);
+	inserterQuery.addBindValue(info.size());
+	inserterQuery.addBindValue(fileType);
+	if(!inserterQuery.exec())
+	{
+		qDebug() << "Failed INSERT query" << inserterQuery.lastError();
+		db.rollback();
+		return DBFAIL;
+	}
+	int lastid = inserterQuery.lastInsertId().toInt();
+	for(PageData &data : pageData)
+	{
+		QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db);
+		contentQuery.addBindValue(lastid);
+		contentQuery.addBindValue(data.pagenumber);
+		contentQuery.addBindValue(data.content);
+		if(!contentQuery.exec())
+		{
+			db.rollback();
+			qDebug() << "Failed content insertion " << contentQuery.lastError();
+			return DBFAIL;
+		}
+	}
+
+	if(!db.commit())
+	{
+		db.rollback();
+		qDebug() << "Failed to commit transaction for " << absPath << " : " << db.lastError();
+		return DBFAIL;
+	}
+	return OK;
+}
+
+int CommandAdd::handle(QStringList arguments)
+{
+	QCommandLineParser parser;
+	parser.addOptions({{{"c", "continue"}, "Continue adding files, don't exit on first error"},
+					   {{"a", "all"}, "On error, no files should be added, even already processed ones"},
+					   {{"v", "verbose"}, "Print skipped and added files"},
+					   {{"n", "threads"}, "Number of threads to use.", "threads"}});
+
+	parser.addHelpOption();
+	parser.addPositionalArgument("paths", "List of paths to process/add to the index", "[paths...]");
+
+	parser.process(arguments);
+	bool keepGoing = false;
+	bool verbose = false;
+	if(parser.isSet("continue"))
+	{
+		keepGoing = true;
+	}
+	if(parser.isSet("verbose"))
+	{
+		verbose = true;
+	}
+	if(parser.isSet("all"))
+	{
+		throw QSSGeneralException("To be implemented");
+	}
+	if(parser.isSet("threads"))
+	{
+		QString threadsCount = parser.value("threads");
+		QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
+	}
+
+	QStringList files = parser.positionalArguments();
+
+	if(files.length() == 0)
+	{
+		QTextStream stream(stdin);
+
+		while(!stream.atEnd())
+		{
+			QString path = stream.readLine();
+			files.append(path);
+		}
+	}
+
+	bool terminate = false;
+	QtConcurrent::blockingMap(files,
+							  [&](QString &path)
+							  {
+								  if(terminate)
+								  {
+									  return;
+								  }
+								  if(verbose)
+								  {
+									  qDebug() << "Processing " << path;
+								  }
+								  auto result = addFile(path);
+								  if(result == DBFAIL)
+								  {
+									  qDebug() << "Failed to add " << path;
+									  if(!keepGoing)
+									  {
+										  terminate = true;
+									  }
+								  }
+								  if(verbose)
+								  {
+									  if(result == SKIPPED)
+									  {
+										  qDebug() << "SKIPPED" << path << "as it already exists in the database";
+									  }
+									  else
+									  {
+										  qDebug() << "Added" << path;
+									  }
+								  }
+							  });
+
+	return 0;
+}