CommandAdd: refactored to use filesaver etc.
This commit is contained in:
parent
c8e958d1a6
commit
f423d0078a
|
|
@ -10,109 +10,8 @@
|
|||
#include <QMutex>
|
||||
#include <QMutexLocker>
|
||||
#include <QtConcurrent/QtConcurrentMap>
|
||||
#include "processor.h"
|
||||
#include "pdfprocessor.h"
|
||||
#include "commandadd.h"
|
||||
#include "defaulttextprocessor.h"
|
||||
#include "tagstripperprocessor.h"
|
||||
#include "nothingprocessor.h"
|
||||
#include "odtprocessor.h"
|
||||
#include "odsprocessor.h"
|
||||
#include "utils.h"
|
||||
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
|
||||
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
|
||||
static NothingProcessor *nothingProcessor = new NothingProcessor();
|
||||
static OdtProcessor *odtProcessor = new OdtProcessor();
|
||||
static OdsProcessor *odsProcessor = new OdsProcessor();
|
||||
|
||||
static QMap<QString, Processor *> processors{
|
||||
{"pdf", new PdfProcessor()}, {"txt", defaultTextProcessor}, {"md", defaultTextProcessor},
|
||||
{"py", defaultTextProcessor}, {"xml", nothingProcessor}, {"html", tagStripperProcessor},
|
||||
{"java", defaultTextProcessor}, {"js", defaultTextProcessor}, {"cpp", defaultTextProcessor},
|
||||
{"c", defaultTextProcessor}, {"sql", defaultTextProcessor}, {"odt", odtProcessor},
|
||||
{"ods", odsProcessor}};
|
||||
|
||||
AddFileResult CommandAdd::addFile(QString path)
|
||||
{
|
||||
QFileInfo info(path);
|
||||
QString absPath = info.absoluteFilePath();
|
||||
auto mtime = info.lastModified().toSecsSinceEpoch();
|
||||
QChar fileType = info.isDir() ? 'd' : 'f';
|
||||
|
||||
QSqlDatabase db = dbConnection();
|
||||
if(fileExistsInDatabase(db, absPath, mtime))
|
||||
{
|
||||
return SKIPPED;
|
||||
}
|
||||
|
||||
Processor *processor = processors.value(info.suffix(), nothingProcessor);
|
||||
QVector<PageData> pageData;
|
||||
if(processor->PREFERED_DATA_SOURCE == FILEPATH)
|
||||
{
|
||||
pageData = processor->process(absPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
pageData = processor->process(Utils::readFile(absPath));
|
||||
}
|
||||
|
||||
// Could happen if a file corrupted for example
|
||||
if(pageData.isEmpty() && processor != nothingProcessor)
|
||||
{
|
||||
Utils::error() << "Could not get any content for " << absPath << endl;
|
||||
}
|
||||
|
||||
// Workaround to "database is locked" error. Perhaps try WAL mode etc.
|
||||
// QMutexLocker locker(&writeMutex);
|
||||
if(!db.transaction())
|
||||
{
|
||||
Utils::error() << "Failed to open transaction for " << absPath << " : " << db.lastError() << endl;
|
||||
return DBFAIL;
|
||||
}
|
||||
|
||||
QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db);
|
||||
delQuery.addBindValue(absPath);
|
||||
if(!delQuery.exec())
|
||||
{
|
||||
Utils::error() << "Failed DELETE query" << delQuery.lastError() << endl;
|
||||
db.rollback();
|
||||
return DBFAIL;
|
||||
}
|
||||
|
||||
QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db);
|
||||
inserterQuery.addBindValue(absPath);
|
||||
inserterQuery.addBindValue(mtime);
|
||||
inserterQuery.addBindValue(info.size());
|
||||
inserterQuery.addBindValue(fileType);
|
||||
if(!inserterQuery.exec())
|
||||
{
|
||||
Utils::error() << "Failed INSERT query" << inserterQuery.lastError() << endl;
|
||||
db.rollback();
|
||||
return DBFAIL;
|
||||
}
|
||||
int lastid = inserterQuery.lastInsertId().toInt();
|
||||
for(PageData &data : pageData)
|
||||
{
|
||||
QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db);
|
||||
contentQuery.addBindValue(lastid);
|
||||
contentQuery.addBindValue(data.pagenumber);
|
||||
contentQuery.addBindValue(data.content);
|
||||
if(!contentQuery.exec())
|
||||
{
|
||||
db.rollback();
|
||||
Utils::error() << "Failed content insertion " << contentQuery.lastError() << endl;
|
||||
return DBFAIL;
|
||||
}
|
||||
}
|
||||
|
||||
if(!db.commit())
|
||||
{
|
||||
db.rollback();
|
||||
Utils::error() << "Failed to commit transaction for " << absPath << " : " << db.lastError() << endl;
|
||||
return DBFAIL;
|
||||
}
|
||||
return OK;
|
||||
}
|
||||
#include "logger.h"
|
||||
|
||||
int CommandAdd::handle(QStringList arguments)
|
||||
{
|
||||
|
|
@ -120,22 +19,14 @@ int CommandAdd::handle(QStringList arguments)
|
|||
parser.addOptions({{{"c", "continue"}, "Continue adding files, don't exit on first error"},
|
||||
{{"a", "all"}, "On error, no files should be added, even already processed ones"},
|
||||
{{"v", "verbose"}, "Print skipped and added files"},
|
||||
{{"n", "threads"}, "Number of threads to use.", "threads"}});
|
||||
{{"t", "threads"}, "Number of threads to use.", "threads"}});
|
||||
|
||||
parser.addHelpOption();
|
||||
parser.addPositionalArgument("add", "Add paths to the index", "add [paths...]");
|
||||
|
||||
parser.process(arguments);
|
||||
bool keepGoing = false;
|
||||
bool verbose = false;
|
||||
if(parser.isSet("continue"))
|
||||
{
|
||||
keepGoing = true;
|
||||
}
|
||||
if(parser.isSet("verbose"))
|
||||
{
|
||||
verbose = true;
|
||||
}
|
||||
bool keepGoing = parser.isSet("continue");
|
||||
bool verbose = parser.isSet("verbose");
|
||||
if(parser.isSet("all"))
|
||||
{
|
||||
throw QSSGeneralException("To be implemented");
|
||||
|
|
@ -159,40 +50,12 @@ int CommandAdd::handle(QStringList arguments)
|
|||
}
|
||||
}
|
||||
|
||||
bool terminate = false;
|
||||
QtConcurrent::blockingMap(files,
|
||||
[&](QString &path)
|
||||
{
|
||||
if(terminate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if(verbose)
|
||||
{
|
||||
Utils::info() << "Processing " << path << endl;
|
||||
}
|
||||
auto result = addFile(path);
|
||||
if(result == DBFAIL)
|
||||
{
|
||||
Utils::error() << "Failed to add " << path << endl;
|
||||
if(!keepGoing)
|
||||
{
|
||||
terminate = true;
|
||||
}
|
||||
}
|
||||
if(verbose)
|
||||
{
|
||||
if(result == SKIPPED)
|
||||
{
|
||||
Utils::info()
|
||||
<< "Skipped" << path << "as it already exists in the database" << endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
Utils::info() << "Added" << path << endl;
|
||||
}
|
||||
}
|
||||
});
|
||||
FileSaver saver(*this->dbService);
|
||||
if(!saver.addFiles(files.toVector(), keepGoing, verbose))
|
||||
{
|
||||
Logger::error() << "Errors occured while trying to add files to the database" << endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue