looqs/cli/filesaver.cpp

#include <QSqlError>
#include <QDateTime>
#include <QtConcurrentMap>
#include <functional>
#include "filesaver.h"
#include "processor.h"
#include "pdfprocessor.h"
#include "commandadd.h"
#include "defaulttextprocessor.h"
#include "tagstripperprocessor.h"
#include "nothingprocessor.h"
#include "odtprocessor.h"
#include "odsprocessor.h"
#include "utils.h"
#include "logger.h"
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
static NothingProcessor *nothingProcessor = new NothingProcessor();
static OdtProcessor *odtProcessor = new OdtProcessor();
static OdsProcessor *odsProcessor = new OdsProcessor();

static QMap<QString, Processor*> processors{
    {"pdf", new PdfProcessor() },
    {"txt", defaultTextProcessor},
    {"md", defaultTextProcessor},
    {"py", defaultTextProcessor},
    {"xml", nothingProcessor},
    {"html", tagStripperProcessor},
    {"java", defaultTextProcessor},
    {"js", defaultTextProcessor},
    {"cpp", defaultTextProcessor},
    {"c", defaultTextProcessor},
    {"sql", defaultTextProcessor},
    {"odt", odtProcessor},
    {"ods", odsProcessor}
};


FileSaver::FileSaver(SqliteDbService &dbService)
{
    this->dbService = &dbService;
}

SaveFileResult FileSaver::addFile(QString path)
{
    QFileInfo info(path);
    QString absPath = info.absoluteFilePath();
    auto mtime = info.lastModified().toSecsSinceEpoch();
    if(this->dbService->fileExistsInDatabase(absPath, mtime))
    {
        return SKIPPED;
    }
    return saveFile(info);
}

SaveFileResult FileSaver::updateFile(QString path)
{
    QFileInfo info(path);
    return saveFile(info);
}

int FileSaver::addFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
{
    return processFiles(paths, std::bind(&FileSaver::addFile, this, std::placeholders::_1), keepGoing, verbose);
}

int FileSaver::updateFiles(const QVector<QString> paths, bool keepGoing, bool verbose)
{
    return processFiles(paths, std::bind(&FileSaver::updateFile, this, std::placeholders::_1), keepGoing, verbose);
}

int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFileResult(QString path)> saverFunc, bool keepGoing, bool verbose)
{
    std::atomic<bool> terminate { false };
    std::atomic<int> processedCount { 0 };
    QtConcurrent::blockingMap(paths, [&](const QString &path) {
        if(terminate.load())
        {
            return;
        }
        if(verbose)
        {
            Logger::info() << "Processing " << path << endl;
        }
        SaveFileResult result = saverFunc(path);
        if(result == DBFAIL || result == PROCESSFAIL)
        {
            Logger::error() << "Failed to process " << path << endl;
            if(!keepGoing)
            {
               terminate = true;
            }
        }
        else
        {
            ++processedCount;
            if(verbose)
            {
                if(result == SKIPPED)
                {
                    Logger::info() << "Skipped" << path << "as it already exists in the database" << endl;
                }
                else if(result == OK)
                {
                    Logger::info() << "Added" << path << endl;
                }
            }
        }

     });
    return processedCount.load();
}


SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
{
    Processor *processor = processors.value(fileInfo.suffix(), nothingProcessor);
    QVector<PageData> pageData;
    QString absPath = fileInfo.absoluteFilePath();

    if(fileInfo.isFile())
    {
        try
        {
            if(processor->PREFERED_DATA_SOURCE == FILEPATH)
            {
                pageData = processor->process(absPath);
            }
            else
            {
                pageData = processor->process(Utils::readFile(absPath));
            }
        }
        catch(LooqsGeneralException &e)
        {
            Logger::error() << "Error while processing" << absPath << ":" << e.message << endl;
            return PROCESSFAIL;
        }
    }


    //Could happen if a file corrupted for example
    if(pageData.isEmpty() && processor != nothingProcessor)
    {
        Logger::error() << "Could not get any content for " << absPath << endl;
    }

    return this->dbService->saveFile(fileInfo, pageData);

}