looqs/cli/commandadd.cpp

#include <QFileInfo>
#include <QDebug>
#include <QSqlQuery>
#include <QSqlError>
#include <QDateTime>
#include <QMap>
#include <QTextStream>
#include <QException>
#include <QCommandLineParser>
#include <QMutex>
#include <QMutexLocker>
#include <QtConcurrent/QtConcurrentMap>
#include "processor.h"
#include "pdfprocessor.h"
#include "commandadd.h"
#include "defaulttextprocessor.h"
#include "tagstripperprocessor.h"
#include "nothingprocessor.h"
#include "odtprocessor.h"
#include "odsprocessor.h"
#include "utils.h"
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
static NothingProcessor *nothingProcessor = new NothingProcessor();
static OdtProcessor *odtProcessor = new OdtProcessor();
static OdsProcessor *odsProcessor = new OdsProcessor();

static QMap<QString, Processor*> processors{
    {"pdf", new PdfProcessor() },
    {"txt", defaultTextProcessor},
    {"md", defaultTextProcessor},
    {"py", defaultTextProcessor},
    {"xml", nothingProcessor},
    {"html", tagStripperProcessor},
    {"java", defaultTextProcessor},
    {"js", defaultTextProcessor},
    {"cpp", defaultTextProcessor},
    {"c", defaultTextProcessor},
    {"sql", defaultTextProcessor},
    {"odt", odtProcessor},
    {"ods", odsProcessor}
};



AddFileResult CommandAdd::addFile(QString path)
{
    QFileInfo info(path);
    QString absPath = info.absoluteFilePath();
    auto mtime = info.lastModified().toSecsSinceEpoch();
    QChar fileType = info.isDir() ? 'd' : 'f';

    QSqlDatabase db = dbConnection();
    if(fileExistsInDatabase(db, absPath, mtime))
    {
        return SKIPPED;
    }



    Processor *processor = processors.value(info.suffix(), nothingProcessor);
    QVector<PageData> pageData;
    if(processor->PREFERED_DATA_SOURCE == FILEPATH)
    {
        pageData = processor->process(absPath);
    }
    else
    {
        pageData = processor->process(Utils::readFile(absPath));
    }

    if(pageData.isEmpty())
    {
        qDebug() << "Could not get any content for " << absPath;
    }


    //Workaround to "database is locked" error. Perhaps try WAL mode etc.
    //QMutexLocker locker(&writeMutex);
    if(!db.transaction())
    {
        qDebug() << "Failed to open transaction for " << absPath << " : " << db.lastError();
        return DBFAIL;
    }

    QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db);
    delQuery.addBindValue(absPath);
    if(!delQuery.exec())
    {
        qDebug() << "Failed DELETE query" <<  delQuery.lastError();
        db.rollback();
        return DBFAIL;
    }


    QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db);
    inserterQuery.addBindValue(absPath);
    inserterQuery.addBindValue(mtime);
    inserterQuery.addBindValue(info.size());
    inserterQuery.addBindValue(fileType);
    if(!inserterQuery.exec())
    {
        qDebug() << "Failed INSERT query" <<  inserterQuery.lastError();
        db.rollback();
        return DBFAIL;
    }
    int lastid = inserterQuery.lastInsertId().toInt();
    for(PageData &data : pageData)
    {
        QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db);
        contentQuery.addBindValue(lastid);
        contentQuery.addBindValue(data.pagenumber);
        contentQuery.addBindValue(data.content);
        if(!contentQuery.exec())
        {
            db.rollback();
            qDebug() << "Failed content insertion " << contentQuery.lastError();
            return DBFAIL;
        }
    }

    if(!db.commit())
    {
        db.rollback();
        qDebug() << "Failed to commit transaction for " << absPath <<  " : " << db.lastError();
        return DBFAIL;
    }
    return OK;
}


int CommandAdd::handle(QStringList arguments)
{
    QCommandLineParser parser;
    parser.addOptions({
                          { { "c", "continue" }, "Continue adding files, don't exit on first error"},
                          { { "a", "all" }, "On error, no files should be added, even already processed ones" },
                          { { "v", "verbose" }, "Print skipped and added files" },
                          { { "n", "threads" }, "Number of threads to use.", "threads" }
                      });

    parser.addHelpOption();
    parser.addPositionalArgument("paths", "List of paths to process/add to the index", "[paths...]");

    parser.process(arguments);
    bool keepGoing = false;
    bool verbose = false;
    if(parser.isSet("continue"))
    {
        keepGoing = true;
    }
    if(parser.isSet("verbose"))
    {
        verbose = true;
    }
    if(parser.isSet("all"))
    {
        throw QSSGeneralException("To be implemented");
    }
    if(parser.isSet("threads"))
    {
        QString threadsCount = parser.value("threads");
        QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
    }

    QStringList files = parser.positionalArguments();

    if(files.length() == 0)
    {
        QTextStream stream(stdin);

        while(!stream.atEnd())
        {
            QString path = stream.readLine();
            files.append(path);
         }
    }

    bool terminate = false;
    QtConcurrent::blockingMap(files, [&](QString &path) {
        if(terminate)
        {
            return;
        }
        if(verbose)
        {
            qDebug() << "Processing " << path;
        }
        auto result = addFile(path);
        if(result == DBFAIL)
        {
            qDebug() << "Failed to add " << path;
            if(!keepGoing)
            {
                terminate = true;
            }
        }
        if(verbose)
        {
            if(result == SKIPPED)
            {
                qDebug() << "SKIPPED" << path << "as it already exists in the database";
            }
            else
            {
                qDebug() << "Added" << path;
            }
        }


    });

    return 0;
}