Added FileSaver for Saving/Processing of files, used by CommandAdd/CommandUpdates
This commit is contained in:
parent
2f1f41608f
commit
c248cf8368
147
cli/filesaver.cpp
Normal file
147
cli/filesaver.cpp
Normal file
@ -0,0 +1,147 @@
|
||||
#include <QSqlError>
|
||||
#include <QDateTime>
|
||||
#include <QtConcurrentMap>
|
||||
#include <functional>
|
||||
#include "filesaver.h"
|
||||
#include "processor.h"
|
||||
#include "pdfprocessor.h"
|
||||
#include "commandadd.h"
|
||||
#include "defaulttextprocessor.h"
|
||||
#include "tagstripperprocessor.h"
|
||||
#include "nothingprocessor.h"
|
||||
#include "odtprocessor.h"
|
||||
#include "odsprocessor.h"
|
||||
#include "utils.h"
|
||||
#include "logger.h"
|
||||
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
|
||||
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
|
||||
static NothingProcessor *nothingProcessor = new NothingProcessor();
|
||||
static OdtProcessor *odtProcessor = new OdtProcessor();
|
||||
static OdsProcessor *odsProcessor = new OdsProcessor();
|
||||
|
||||
static QMap<QString, Processor*> processors{
|
||||
{"pdf", new PdfProcessor() },
|
||||
{"txt", defaultTextProcessor},
|
||||
{"md", defaultTextProcessor},
|
||||
{"py", defaultTextProcessor},
|
||||
{"xml", nothingProcessor},
|
||||
{"html", tagStripperProcessor},
|
||||
{"java", defaultTextProcessor},
|
||||
{"js", defaultTextProcessor},
|
||||
{"cpp", defaultTextProcessor},
|
||||
{"c", defaultTextProcessor},
|
||||
{"sql", defaultTextProcessor},
|
||||
{"odt", odtProcessor},
|
||||
{"ods", odsProcessor}
|
||||
};
|
||||
|
||||
|
||||
FileSaver::FileSaver(SqliteDbService &dbService)
|
||||
{
|
||||
this->dbService = &dbService;
|
||||
}
|
||||
|
||||
SaveFileResult FileSaver::addFile(QString path)
|
||||
{
|
||||
QFileInfo info(path);
|
||||
QString absPath = info.absoluteFilePath();
|
||||
auto mtime = info.lastModified().toSecsSinceEpoch();
|
||||
if(this->dbService->fileExistsInDatabase(absPath, mtime))
|
||||
{
|
||||
return SKIPPED;
|
||||
}
|
||||
return saveFile(info);
|
||||
}
|
||||
|
||||
SaveFileResult FileSaver::updateFile(QString path)
|
||||
{
|
||||
QFileInfo info(path);
|
||||
return saveFile(info);
|
||||
}
|
||||
|
||||
bool FileSaver::addFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
|
||||
{
|
||||
return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose);
|
||||
}
|
||||
|
||||
bool FileSaver::updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
|
||||
{
|
||||
return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose);
|
||||
}
|
||||
|
||||
bool FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc, bool keepGoing, bool verbose)
|
||||
{
|
||||
std::atomic<bool> terminate { false };
|
||||
std::atomic<bool> hasError { false };
|
||||
QtConcurrent::blockingMap(paths, [&](const QString &path) {
|
||||
if(terminate.load())
|
||||
{
|
||||
return;
|
||||
}
|
||||
if(verbose)
|
||||
{
|
||||
Logger::info() << "Processing " << path << endl;
|
||||
}
|
||||
SaveFileResult result = saverFunc(path);
|
||||
if(result == DBFAIL || result == PROCESSFAIL)
|
||||
{
|
||||
hasError = true;
|
||||
Logger::error() << "Failed to process " << path << endl;
|
||||
if(!keepGoing)
|
||||
{
|
||||
terminate = true;
|
||||
|
||||
}
|
||||
}
|
||||
if(verbose)
|
||||
{
|
||||
if(result == SKIPPED)
|
||||
{
|
||||
Logger::info() << "Skipped" << path << "as it already exists in the database" << endl;
|
||||
}
|
||||
else if(result == OK)
|
||||
{
|
||||
Logger::info() << "Added" << path << endl;
|
||||
}
|
||||
}
|
||||
});
|
||||
return ! hasError.load();
|
||||
}
|
||||
|
||||
|
||||
SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
|
||||
{
|
||||
Processor *processor = processors.value(fileInfo.suffix(), nothingProcessor);
|
||||
QVector<PageData> pageData;
|
||||
QString absPath = fileInfo.absoluteFilePath();
|
||||
|
||||
if(fileInfo.isFile())
|
||||
{
|
||||
try
|
||||
{
|
||||
if(processor->PREFERED_DATA_SOURCE == FILEPATH)
|
||||
{
|
||||
pageData = processor->process(absPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
pageData = processor->process(Utils::readFile(absPath));
|
||||
}
|
||||
}
|
||||
catch(QSSGeneralException &e)
|
||||
{
|
||||
Logger::error() << "Error while processing" << absPath << ":" << e.message;
|
||||
return PROCESSFAIL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//Could happen if a file corrupted for example
|
||||
if(pageData.isEmpty() && processor != nothingProcessor)
|
||||
{
|
||||
Logger::error() << "Could not get any content for " << absPath << endl;
|
||||
}
|
||||
|
||||
return this->dbService->saveFile(fileInfo, pageData);
|
||||
|
||||
}
|
28
cli/filesaver.h
Normal file
28
cli/filesaver.h
Normal file
@ -0,0 +1,28 @@
|
||||
#ifndef FILESAVER_H
|
||||
#define FILESAVER_H
|
||||
#include <QSqlDatabase>
|
||||
#include <QFileInfo>
|
||||
#include "command.h"
|
||||
#include "pagedata.h"
|
||||
#include "filedata.h"
|
||||
#include "sqlitedbservice.h"
|
||||
|
||||
|
||||
class FileSaver
|
||||
{
|
||||
private:
|
||||
SqliteDbService *dbService;
|
||||
protected:
|
||||
SaveFileResult addFile(QString path);
|
||||
SaveFileResult updateFile(QString path);
|
||||
public:
|
||||
FileSaver(SqliteDbService &dbService);
|
||||
SaveFileResult saveFile(const QFileInfo &fileInfo);
|
||||
bool processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc, bool keepGoing, bool verbose);
|
||||
bool addFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
|
||||
bool updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose);
|
||||
|
||||
;
|
||||
};
|
||||
|
||||
#endif // FILESAVER_H
|
Loading…
Reference in New Issue
Block a user