crtxcr/looqs
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

cli: Move most classes to shared lib for reuse

Browse Source
This commit is contained in:
Albert S.
2022-03-18 22:33:45 +01:00
parent d43c35819d
commit 478d57b342
33 changed files with 32 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
cli/cli.pro
View File

@ -14,56 +14,28 @@ DEFINES += QT_DEPRECATED_WARNINGS
# In order to do so, uncomment the following line.
# You can also select to disable deprecated APIs only up to a certain version of Qt.
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0
LIBS += -luchardet -lpoppler-qt5 -lquazip5
SOURCES += \
main.cpp \
encodingdetector.cpp \
pagedata.cpp \
processor.cpp \
pdfprocessor.cpp \
defaulttextprocessor.cpp \
commandadd.cpp \
sandboxedprocessor.cpp \
tagstripperprocessor.cpp \
nothingprocessor.cpp \
odtprocessor.cpp \
utils.cpp \
odsprocessor.cpp \
commanddelete.cpp \
commandupdate.cpp \
filesaver.cpp \
sqlitedbservice.cpp \
commandsearch.cpp \
commandlist.cpp
HEADERS += \
encodingdetector.h \
processor.h \
pagedata.h \
pdfprocessor.h \
defaulttextprocessor.h \
command.h \
commandadd.h \
sandboxedprocessor.h \
tagstripperprocessor.h \
nothingprocessor.h \
odtprocessor.h \
utils.h \
odsprocessor.h \
commanddelete.h \
commandupdate.h \
filesaver.h \
sqlitedbservice.h \
commandsearch.h \
commandlist.h
INCLUDEPATH += /usr/include/poppler/qt5/ /usr/include/quazip5
win32:CONFIG(release, debug|release): LIBS += -L$$OUT_PWD/../shared/release/ -lshared
else:win32:CONFIG(debug, debug|release): LIBS += -L$$OUT_PWD/../shared/debug/ -lshared
else:unix: LIBS += -L$$OUT_PWD/../shared/ -lshared
LIBS += -luchardet -lpoppler-qt5 -lquazip5
INCLUDEPATH += $$PWD/../shared
DEPENDPATH += $$PWD/../shared

2
cli/command.h
View File

@ -1,8 +1,6 @@
#ifndef COMMAND_H
#define COMMAND_H
#include <QStringList>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QThreadStorage>
#include <QVariant>
#include "utils.h"

2
cli/commandadd.cpp
View File

@ -1,7 +1,5 @@
#include <QFileInfo>
#include <QDebug>
#include <QSqlQuery>
#include <QSqlError>
#include <QDateTime>
#include <QMap>
#include <QTextStream>

1
cli/commanddelete.cpp
View File

@ -3,7 +3,6 @@
#include <QFileInfo>
#include <QDebug>
#include <QRegularExpression>
#include <QSqlError>
#include "commanddelete.h"
#include "logger.h"

1
cli/commandupdate.cpp
View File

@ -2,7 +2,6 @@
#include <QFileInfo>
#include <QDateTime>
#include <QThreadPool>
#include <QtConcurrentRun>
#include "commandupdate.h"
#include "logger.h"

30
cli/defaulttextprocessor.cpp
View File

@ -1,30 +0,0 @@
#include <QFile>
#include <QDataStream>
#include <QTextCodec>
#include <QDebug>
#include "defaulttextprocessor.h"
DefaultTextProcessor::DefaultTextProcessor()
{
}
QString DefaultTextProcessor::processText(const QByteArray &data) const
{
QString encoding = encodingDetector.detectEncoding(data);
if(!encoding.isEmpty())
{
QTextCodec *codec = QTextCodec::codecForName(encoding.toUtf8());
if(codec != nullptr)
{
return codec->toUnicode(data);
}
qWarning() << "No codec found for " << encoding;
return QString(data);
}
return {};
}
QVector<PageData> DefaultTextProcessor::process(const QByteArray &data) const
{
return {{0, processText(data)}};
}

17
cli/defaulttextprocessor.h
View File

@ -1,17 +0,0 @@
#ifndef DEFAULTTEXTPROCESSOR_H
#define DEFAULTTEXTPROCESSOR_H
#include "processor.h"
#include "encodingdetector.h"
class DefaultTextProcessor : public Processor
{
protected:
EncodingDetector encodingDetector;
public:
DefaultTextProcessor();
QString processText(const QByteArray &data) const;
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // DEFAULTTEXTPROCESSOR_H

45
cli/encodingdetector.cpp
View File

@ -1,45 +0,0 @@
#include <QDataStream>
#include "encodingdetector.h"
#include <looqsgeneralexception.h>
EncodingDetector::EncodingDetector()
{
}
QString EncodingDetector::detectEncoding(const QByteArray &data) const
{
uchardet_t detector = uchardet_new();
if(uchardet_handle_data(detector, data.data(), data.size()) != 0)
{
uchardet_delete(detector);
throw LooqsGeneralException("Decoder failed");
}
uchardet_data_end(detector);
QString encoding = uchardet_get_charset(detector);
uchardet_delete(detector);
return encoding;
}
QString EncodingDetector::detectEncoding(QDataStream &s) const
{
uchardet_t detector = uchardet_new();
char buffer[4096];
int n;
while((n = s.readRawData(buffer, sizeof(buffer))) > 0)
{
if(uchardet_handle_data(detector, buffer, n) != 0)
{
uchardet_delete(detector);
throw LooqsGeneralException("Decoder failed");
}
}
if(n == -1)
{
uchardet_delete(detector);
throw LooqsGeneralException("Read failed");
}
uchardet_data_end(detector);
QString encoding = uchardet_get_charset(detector);
uchardet_delete(detector);
return encoding;
}

14
cli/encodingdetector.h
View File

@ -1,14 +0,0 @@
#ifndef ENCODINGDETECTOR_H
#define ENCODINGDETECTOR_H
#include <QString>
#include <uchardet/uchardet.h>
class EncodingDetector
{
public:
EncodingDetector();
QString detectEncoding(const QByteArray &data) const;
QString detectEncoding(QDataStream &s) const;
};
#endif // ENCODINGDETECTOR_H

144
cli/filesaver.cpp
View File

@ -1,144 +0,0 @@
#include <QSqlError>
#include <QDateTime>
#include <QtConcurrentMap>
#include <QProcess>
#include <functional>
#include "filesaver.h"
#include "processor.h"
#include "pdfprocessor.h"
#include "commandadd.h"
#include "defaulttextprocessor.h"
#include "tagstripperprocessor.h"
#include "nothingprocessor.h"
#include "odtprocessor.h"
#include "odsprocessor.h"
#include "utils.h"
#include "logger.h"
FileSaver::FileSaver(SqliteDbService &dbService)
{
this->dbService = &dbService;
}
SaveFileResult FileSaver::addFile(QString path)
{
QFileInfo info(path);
QString absPath = info.absoluteFilePath();
auto mtime = info.lastModified().toSecsSinceEpoch();
if(this->dbService->fileExistsInDatabase(absPath, mtime))
{
return SKIPPED;
}
return saveFile(info);
}
SaveFileResult FileSaver::updateFile(QString path)
{
QFileInfo info(path);
return saveFile(info);
}
int FileSaver::addFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
{
return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose);
}
int FileSaver::updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
{
return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose);
}
int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
bool keepGoing, bool verbose)
{
std::atomic<bool> terminate{false};
std::atomic<int> processedCount{0};
QtConcurrent::blockingMap(paths,
[&](const QString &path)
{
if(terminate.load())
{
return;
}
if(verbose)
{
Logger::info() << "Processing " << path << Qt::endl;
}
SaveFileResult result = saverFunc(path);
if(result == DBFAIL || result == PROCESSFAIL)
{
Logger::error() << "Failed to process " << path << Qt::endl;
if(!keepGoing)
{
terminate = true;
}
}
else
{
++processedCount;
if(verbose)
{
if(result == SKIPPED)
{
Logger::info() << "Skipped" << path
<< "as it already exists in the database" << Qt::endl;
}
else if(result == OK)
{
Logger::info() << "Added" << path << Qt::endl;
}
}
}
});
return processedCount.load();
}
SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
{
QVector<PageData> pageData;
QString absPath = fileInfo.absoluteFilePath();
int status = -1;
if(fileInfo.isFile())
{
QProcess process;
QStringList args;
args << "process" << absPath;
process.setProcessChannelMode(QProcess::ForwardedErrorChannel);
process.start("/proc/self/exe", args);
process.waitForStarted();
process.waitForFinished();
/* TODO: This is suboptimal as it eats lots of mem
* but avoids a weird QDataStream/QProcess behaviour
* where it thinks the process has ended when it has not...
*
* Also, there seem to be issues with reads not being blocked, so
* the only reliable way appears to be waiting until the process
* finishes.
*/
QDataStream in(process.readAllStandardOutput());
while(!in.atEnd())
{
PageData pd;
in >> pd;
pageData.append(pd);
}
status = process.exitCode();
if(status != 0)
{
Logger::error() << "Error while processing" << absPath << ":"
<< "Exit code " << status << Qt::endl;
return PROCESSFAIL;
}
}
// Could happen if a file corrupted for example
if(pageData.isEmpty() && status != NOTHING_PROCESSED)
{
Logger::error() << "Could not get any content for " << absPath << Qt::endl;
}
return this->dbService->saveFile(fileInfo, pageData);
}

30
cli/filesaver.h
View File

@ -1,30 +0,0 @@
#ifndef FILESAVER_H
#define FILESAVER_H
#include <QSqlDatabase>
#include <QFileInfo>
#include "command.h"
#include "pagedata.h"
#include "filedata.h"
#include "sqlitedbservice.h"
class FileSaver
{
private:
SqliteDbService *dbService;
protected:
SaveFileResult addFile(QString path);
SaveFileResult updateFile(QString path);
public:
FileSaver(SqliteDbService &dbService);
SaveFileResult saveFile(const QFileInfo &fileInfo);
int processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc,
bool keepGoing, bool verbose);
int addFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
int updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
;
};
#endif // FILESAVER_H

3
cli/main.cpp
View File

@ -5,9 +5,6 @@
#include <QDataStream>
#include <QDebug>
#include <QProcessEnvironment>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QMap>
#include <QDebug>
#include <QSettings>

5
cli/nothingprocessor.cpp
View File

@ -1,5 +0,0 @@
#include "nothingprocessor.h"
NothingProcessor::NothingProcessor()
{
}

19
cli/nothingprocessor.h
View File

@ -1,19 +0,0 @@
#ifndef NOTHINGPROCESSOR_H
#define NOTHINGPROCESSOR_H
#include <QVector>
#include "processor.h"
#include "pagedata.h"
class NothingProcessor : public Processor
{
public:
NothingProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override
{
return {};
}
};
#endif // NOTHINGPROCESSOR_H

5
cli/odsprocessor.cpp
View File

@ -1,5 +0,0 @@
#include "odsprocessor.h"
OdsProcessor::OdsProcessor()
{
}

10
cli/odsprocessor.h
View File

@ -1,10 +0,0 @@
#ifndef ODSPROCESSOR_H
#define ODSPROCESSOR_H
#include "odtprocessor.h"
class OdsProcessor : public OdtProcessor
{
public:
OdsProcessor();
};
#endif // ODSPROCESSOR_H

26
cli/odtprocessor.cpp
View File

@ -1,26 +0,0 @@
#include <quazip5/quazip.h>
#include <quazip5/quazipfile.h>
#include "odtprocessor.h"
#include "tagstripperprocessor.h"
QVector<PageData> OdtProcessor::process(const QByteArray &data) const
{
throw LooqsGeneralException("Not implemented yet");
}
QVector<PageData> OdtProcessor::process(QString path) const
{
QuaZipFile zipFile(path);
zipFile.setFileName("content.xml");
if(!zipFile.open(QIODevice::ReadOnly))
{
throw LooqsGeneralException("Error while opening file " + path);
}
QByteArray entireContent = zipFile.readAll();
if(entireContent.isEmpty())
{
throw LooqsGeneralException("Error while reading content.xml of " + path);
}
TagStripperProcessor tsp;
return tsp.process(entireContent);
}

16
cli/odtprocessor.h
View File

@ -1,16 +0,0 @@
#ifndef ODTPROCESSOR_H
#define ODTPROCESSOR_H
#include "processor.h"
class OdtProcessor : public Processor
{
public:
OdtProcessor()
{
this->PREFERED_DATA_SOURCE = FILEPATH;
}
QVector<PageData> process(const QByteArray &data) const override;
QVector<PageData> process(QString path) const override;
};
#endif // ODTPROCESSOR_H

13
cli/pagedata.cpp
View File

@ -1,13 +0,0 @@
#include "pagedata.h"
QDataStream &operator<<(QDataStream &out, const PageData &pd)
{
out << pd.pagenumber << pd.content;
return out;
}
QDataStream &operator>>(QDataStream &in, PageData &pd)
{
in >> pd.pagenumber >> pd.content;
return in;
}

29
cli/pagedata.h
View File

@ -1,29 +0,0 @@
#ifndef PAGEDATA_H
#define PAGEDATA_H
#include <QString>
#include <QMetaType>
#include <QDataStream>
class PageData
{
public:
unsigned int pagenumber = 0;
QString content;
PageData()
{
}
PageData(unsigned int pagenumber, QString content)
{
this->pagenumber = pagenumber;
this->content = content;
}
};
Q_DECLARE_METATYPE(PageData);
QDataStream &operator<<(QDataStream &out, const PageData &pd);
QDataStream &operator>>(QDataStream &in, PageData &pd);
#endif // PAGEDATA_H

37
cli/pdfprocessor.cpp
View File

@ -1,37 +0,0 @@
#include <QScopedPointer>
#include <poppler-qt5.h>
#include "pdfprocessor.h"
PdfProcessor::PdfProcessor()
{
}
QVector<PageData> PdfProcessor::process(const QByteArray &data) const
{
QVector<PageData> result;
QScopedPointer<Poppler::Document> doc(Poppler::Document::loadFromData(data));
if(doc.isNull())
{
throw LooqsGeneralException("Failed to process pdf data");
}
if(doc->isLocked())
{
throw LooqsGeneralException("Doc is locked");
}
QRectF entirePage;
auto pagecount = doc->numPages();
QString entire;
entire.reserve(data.size()); // TODO too much
for(auto i = 0; i < pagecount; i++)
{
QString text = doc->page(i)->text(entirePage);
result.append({static_cast<unsigned int>(i + 1), text});
/*TODO: hack, so we can fts search several words over the whole document, not just pages.
* this of course uses more space and should be solved differently.
*/
entire += text;
}
result.append({0, entire});
return result;
}

13
cli/pdfprocessor.h
View File

@ -1,13 +0,0 @@
#ifndef PDFPROCESSOR_H
#define PDFPROCESSOR_H
#include "processor.h"
class PdfProcessor : public Processor
{
public:
PdfProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // PDFPROCESSOR_H

5
cli/processor.cpp
View File

@ -1,5 +0,0 @@
#include "processor.h"
Processor::Processor()
{
}

34
cli/processor.h
View File

@ -1,34 +0,0 @@
#ifndef PROCESSOR_H
#define PROCESSOR_H
#include <QVector>
#include <QFile>
#include "pagedata.h"
#include "utils.h"
enum DataSource
{
FILEPATH,
ARRAY
};
#define NOTHING_PROCESSED 4
class Processor
{
public:
/* Indicates the data source the processor performs best with. For example,
* you do not want to read the entire of a compressed archive just to get the content of
* a single file */
DataSource PREFERED_DATA_SOURCE = ARRAY;
Processor();
virtual QVector<PageData> process(const QByteArray &data) const = 0;
virtual QVector<PageData> process(QString path) const
{
return process(Utils::readFile(path));
}
virtual ~Processor()
{
}
};
#endif // PROCESSOR_H

111
cli/sandboxedprocessor.cpp
View File

@ -1,111 +0,0 @@
#include <QFile>
#include <QFileInfo>
#include <QDataStream>
#include "sandboxedprocessor.h"
#include "pdfprocessor.h"
#include "defaulttextprocessor.h"
#include "tagstripperprocessor.h"
#include "nothingprocessor.h"
#include "odtprocessor.h"
#include "odsprocessor.h"
#include "../submodules/exile.h/exile.h"
#include "logger.h"
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
static NothingProcessor *nothingProcessor = new NothingProcessor();
static OdtProcessor *odtProcessor = new OdtProcessor();
static OdsProcessor *odsProcessor = new OdsProcessor();
static QMap<QString, Processor *> processors{
{"pdf", new PdfProcessor()}, {"txt", defaultTextProcessor}, {"md", defaultTextProcessor},
{"py", defaultTextProcessor}, {"xml", nothingProcessor}, {"html", tagStripperProcessor},
{"java", defaultTextProcessor}, {"js", defaultTextProcessor}, {"cpp", defaultTextProcessor},
{"c", defaultTextProcessor}, {"sql", defaultTextProcessor}, {"odt", odtProcessor},
{"ods", odsProcessor}};
void SandboxedProcessor::enableSandbox(QString readablePath)
{
struct exile_policy *policy = exile_init_policy();
if(policy == NULL)
{
qCritical() << "Could not init exile";
exit(EXIT_FAILURE);
}
policy->namespace_options = EXILE_UNSHARE_NETWORK | EXILE_UNSHARE_USER;
if(!readablePath.isEmpty())
{
std::string readablePathLocation = readablePath.toStdString();
if(exile_append_path_policy(policy, EXILE_FS_ALLOW_ALL_READ, readablePathLocation.c_str()) != 0)
{
qCritical() << "Failed to add path policies";
exit(EXIT_FAILURE);
}
}
else
{
policy->no_fs = 1;
}
int ret = exile_enable_policy(policy);
if(ret != 0)
{
qDebug() << "Failed to establish sandbox: " << ret;
exit(EXIT_FAILURE);
}
exile_free_policy(policy);
}
void SandboxedProcessor::printResults(const QVector<PageData> &pageData)
{
QFile fsstdout;
fsstdout.open(stdout, QIODevice::WriteOnly);
QDataStream stream(&fsstdout);
for(const PageData &data : pageData)
{
stream << data;
// fsstdout.flush();
}
fsstdout.close();
}
int SandboxedProcessor::process()
{
QFileInfo fileInfo(this->filePath);
Processor *processor = processors.value(fileInfo.suffix(), nothingProcessor);
if(processor == nothingProcessor)
{
/* Nothing to do */
return NOTHING_PROCESSED;
}
QVector<PageData> pageData;
QString absPath = fileInfo.absoluteFilePath();
try
{
if(processor->PREFERED_DATA_SOURCE == FILEPATH)
{
/* Read access to FS needed... doh..*/
enableSandbox(absPath);
pageData = processor->process(absPath);
}
else
{
QByteArray data = Utils::readFile(absPath);
enableSandbox();
pageData = processor->process(data);
}
}
catch(LooqsGeneralException &e)
{
Logger::error() << "Error while processing" << absPath << ":" << e.message << Qt::endl;
return 3 /* PROCESSFAIL */;
}
printResults(pageData);
return 0;
}

23
cli/sandboxedprocessor.h
View File

@ -1,23 +0,0 @@
#ifndef SANDBOXEDPROCESSOR_H
#define SANDBOXEDPROCESSOR_H
#include <QString>
#include "pagedata.h"
class SandboxedProcessor
{
private:
QString filePath;
void enableSandbox(QString readablePath = "");
void printResults(const QVector<PageData> &pageData);
public:
SandboxedProcessor(QString filepath)
{
this->filePath = filepath;
}
int process();
};
#endif // SANDBOXEDPROCESSOR_H

172
cli/sqlitedbservice.cpp
View File

@ -1,172 +0,0 @@
#include <QSqlQuery>
#include <QFileInfo>
#include <QDateTime>
#include <QSqlError>
#include "sqlitedbservice.h"
#include "filedata.h"
#include "logger.h"
bool SqliteDbService::fileExistsInDatabase(QString path, qint64 mtime)
{
auto query = QSqlQuery("SELECT 1 FROM file WHERE path = ? and mtime = ?", dbFactory->forCurrentThread());
query.addBindValue(path);
query.addBindValue(mtime);
if(!query.exec())
{
throw LooqsGeneralException("Error while trying to query for file existance: " + query.lastError().text());
}
if(!query.next())
{
return false;
}
return query.value(0).toBool();
}
QVector<SearchResult> SqliteDbService::search(const LooqsQuery &query)
{
auto connection = dbFactory->forCurrentThread();
SqliteSearch searcher(connection);
return searcher.search(query);
}
bool SqliteDbService::fileExistsInDatabase(QString path)
{
auto query = QSqlQuery(dbFactory->forCurrentThread());
query.prepare("SELECT 1 FROM file WHERE path = ?");
query.addBindValue(path);
if(!query.exec())
{
throw LooqsGeneralException("Error while trying to query for file existance: " + query.lastError().text());
}
if(!query.next())
{
return false;
}
return query.value(0).toBool();
}
SqliteDbService::SqliteDbService(DatabaseFactory &dbFactory)
{
this->dbFactory = &dbFactory;
}
bool SqliteDbService::deleteFile(QString path)
{
QSqlQuery query(this->dbFactory->forCurrentThread());
query.prepare("DELETE FROM file WHERE path = ?");
query.addBindValue(path);
bool result = query.exec();
if(!result)
{
Logger::error() << "Failed to delete file" << path << Qt::endl;
}
return result;
}
int SqliteDbService::getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit)
{
int processedRows = 0;
// TODO: translate/convert wildCardPattern to SQL where instead of regex
QString sql = "SELECT path, mtime, size, filetype FROM file";
if(limit != 0)
{
sql += " LIMIT " + QString::number(limit);
}
if(offset != 0)
{
sql += " OFFSET " + QString::number(offset);
}
auto query = QSqlQuery(dbFactory->forCurrentThread());
query.prepare(sql);
query.setForwardOnly(true);
if(!query.exec())
{
throw LooqsGeneralException("Error while trying to retrieve files from database: " + query.lastError().text());
}
// TODO: port this to QRegularExpression once >5.12 gets more widespread because of this bug
// https://bugreports.qt.io/browse/QTBUG-72539?focusedCommentId=439053&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel
bool usePattern = !wildCardPattern.isEmpty();
QRegExp regexPattern(wildCardPattern);
regexPattern.setPatternSyntax(QRegExp::PatternSyntax::WildcardUnix);
while(query.next())
{
QString absPath = query.value(0).toString();
if(!usePattern || regexPattern.exactMatch(absPath))
{
FileData current;
current.absPath = absPath;
current.mtime = query.value(1).toInt();
current.size = query.value(2).toInt();
current.filetype = query.value(3).toChar();
results.append(current);
}
++processedRows;
}
return processedRows;
}
SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData)
{
QString absPath = fileInfo.absoluteFilePath();
auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
QChar fileType = fileInfo.isDir() ? 'd' : 'f';
QSqlDatabase db = dbFactory->forCurrentThread();
QSqlQuery delQuery(db);
delQuery.prepare("DELETE FROM file WHERE path = ?");
delQuery.addBindValue(absPath);
QSqlQuery inserterQuery(db);
inserterQuery.prepare("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)");
inserterQuery.addBindValue(absPath);
inserterQuery.addBindValue(mtime);
inserterQuery.addBindValue(fileInfo.size());
inserterQuery.addBindValue(fileType);
if(!db.transaction())
{
Logger::error() << "Failed to open transaction for " << absPath << " : " << db.lastError() << Qt::endl;
return DBFAIL;
}
if(!delQuery.exec())
{
Logger::error() << "Failed DELETE query" << delQuery.lastError() << Qt::endl;
db.rollback();
return DBFAIL;
}
if(!inserterQuery.exec())
{
Logger::error() << "Failed INSERT query" << inserterQuery.lastError() << Qt::endl;
db.rollback();
return DBFAIL;
}
int lastid = inserterQuery.lastInsertId().toInt();
for(const PageData &data : pageData)
{
QSqlQuery contentQuery(db);
contentQuery.prepare("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)");
contentQuery.addBindValue(lastid);
contentQuery.addBindValue(data.pagenumber);
contentQuery.addBindValue(data.content);
if(!contentQuery.exec())
{
db.rollback();
Logger::error() << "Failed content insertion " << contentQuery.lastError() << Qt::endl;
return DBFAIL;
}
}
if(!db.commit())
{
db.rollback();
Logger::error() << "Failed to commit transaction for " << absPath << " : " << db.lastError() << Qt::endl;
return DBFAIL;
}
return OK;
}

33
cli/sqlitedbservice.h
View File

@ -1,33 +0,0 @@
#ifndef SQLITEDBSERVICE_H
#define SQLITEDBSERVICE_H
#include <QFileInfo>
#include "databasefactory.h"
#include "utils.h"
#include "pagedata.h"
#include "filedata.h"
#include "../shared/sqlitesearch.h"
#include "../shared/token.h"
enum SaveFileResult
{
OK,
SKIPPED,
DBFAIL,
PROCESSFAIL
};
class SqliteDbService
{
private:
DatabaseFactory *dbFactory = nullptr;
public:
SqliteDbService(DatabaseFactory &dbFactory);
SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData);
int getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit);
bool deleteFile(QString path);
bool fileExistsInDatabase(QString path);
bool fileExistsInDatabase(QString path, qint64 mtime);
QVector<SearchResult> search(const LooqsQuery &query);
};
#endif // SQLITEDBSERVICE_H

14
cli/tagstripperprocessor.cpp
View File

@ -1,14 +0,0 @@
#include "tagstripperprocessor.h"
TagStripperProcessor::TagStripperProcessor()
{
}
QVector<PageData> TagStripperProcessor::process(const QByteArray &data) const
{
auto result = DefaultTextProcessor::process(data);
// TODO: does not work properly with <br> and does not deal with entities...
result[0].content.remove(QRegExp("<[^>]*>"));
return result;
}

14
cli/tagstripperprocessor.h
View File

@ -1,14 +0,0 @@
#ifndef XMLSTRIPPERPROCESSOR_H
#define XMLSTRIPPERPROCESSOR_H
#include "defaulttextprocessor.h"
class TagStripperProcessor : public DefaultTextProcessor
{
public:
TagStripperProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // XMLSTRIPPERPROCESSOR_H

20
cli/utils.cpp
View File

@ -1,20 +0,0 @@
#include <QDebug>
#include "utils.h"
Utils::Utils()
{
}
QByteArray Utils::readFile(QString path)
{
QFile file(path);
if(!file.open(QIODevice::ReadOnly))
{
throw LooqsGeneralException("Failed to open file: " + path);
}
QByteArray data = file.readAll();
if(data.isEmpty() && file.error() != QFileDevice::FileError::NoError)
{
throw LooqsGeneralException("Error reading file: " + path + ", Error: " + file.error());
}
return data;
}

17
cli/utils.h
View File

@ -1,17 +0,0 @@
#ifndef UTILS_H
#define UTILS_H
#include <QFile>
#include <QString>
#include <QByteArray>
#include <QTextStream>
#include <QDebug>
#include "looqsgeneralexception.h"
class Utils
{
public:
Utils();
static QByteArray readFile(QString path);
};
#endif // UTILS_H