crtxcr/looqs
1
0
Fork 0
Κώδικας Ζητήματα Pull Requests 1 Κυκλοφορίες Wiki Δραστηριότητα

Begin a C++ cli and remove the Python scripts

Περιήγηση Στο Πηγαίο
Αυτή η υποβολή περιλαμβάνεται σε:
Albert S. 2019-04-06 17:16:42 +02:00
γονέας b7b93a66a9
υποβολή d7f554949f
32 αρχεία άλλαξαν με 817 προσθήκες και 214 διαγραφές
Εμφάνιση Στατιστικών Λήψη Αρχείου Patch Λήψη Αρχείου Diff Ανάπτυξη όλων των αρχείων Σύμπτυξη όλων των αρχείων

167
addindex
Προβολή Αρχείου

@ -1,167 +0,0 @@
#!/usr/bin/python3
import sqlite3
import os.path
import sys
import subprocess
import zipfile
import xml.etree.ElementTree
import re
import chardet
import config
from multiprocessing import Pool
class pagedata:
page = 0
content = ""
def singlepagelist(content):
result = pagedata()
result.content = content
result.page = 0
l = list();
l.append(result)
return l
def striptags(content):
result = ""
try:
result = ''.join(xml.etree.ElementTree.fromstring(content).itertext())
except:
#TODO: test<br>test2 will make it testtest2 not test test2
result = re.sub('<[^>]*>', '', content)
return result
def strip_irrelevant(content):
result = content.replace("\n", " ").replace("\t", " ")
result = re.sub(' +', ' ', result)
return result;
def process_pdf(path):
result = list()
args=["pdftotext", path, "-"]
stdout,stderr = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
content = strip_irrelevant(stdout.decode('utf-8'))
#it is faster to do it like this than to call pdfottext for each page
splitted = content.split("\f")
count=1
for page in splitted:
data = pagedata()
data.page = count
data.content = page
result.append(data)
count += 1
#TODO: current hack, so we can fts search several words over the whole document
#this of course uses more space, but in the end that's not a big problem
#Nevertheless, this remains a hack
everything = pagedata()
everything.page = 0
everything.content = content.replace("\f", "")
result.append(everything)
return result
def process_odt(path):
fd = zipfile.ZipFile(path)
content = fd.read("content.xml").decode("utf-8")
fd.close()
return singlepagelist(striptags(content))
def process_ods(path):
return process_odt(path)
def readtext(path):
fd = open(path, "rb")
content = fd.read()
fd.close()
result=""
try:
result = str(content.decode("utf-8"))
except:
try:
encoding = chardet.detect(content)["encoding"];
if encoding == None:
result = ""
else:
result = str(content.decode(encoding))
except:
print("FAILED DECODING: " + path)
result = ""
return result
def process_striptags(path):
content = readtext(path)
return singlepagelist(striptags(content))
def process_text(path):
return singlepagelist(readtext(path))
def process_nothing(path):
return list()
def exists(abspath, mtime):
cursor = dbcon.cursor()
cursor.execute("SELECT 1 FROM file WHERE path = ? AND mtime = ?" , (abspath, mtime))
result = cursor.fetchone()
if result != None and result[0] == 1:
return True
return False
def insert(path):
print("processing", path)
abspath=os.path.abspath(path)
mtime = int(os.stat(abspath).st_mtime)
if exists(abspath, mtime):
print("Leaving alone " + abspath + " because it wasn't changed")
return
basename=os.path.basename(abspath)
ext = os.path.splitext(abspath)[1]
content=""
processor=process_nothing
if ext in preprocess:
processor=preprocess[ext]
pagedatalist = processor(abspath)
#TODO: assumes sqlitehas been built with thread safety (and it is the default)
cursor = dbcon.cursor()
cursor.execute("BEGIN TRANSACTION")
cursor.execute("DELETE FROM file WHERE path = ?", (abspath,))
cursor.execute("INSERT INTO file(path, mtime) VALUES(?, ?) ", (abspath, mtime))
fileid=cursor.lastrowid
for pagedata in pagedatalist:
cursor.execute("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", (fileid, pagedata.page, pagedata.content))
cursor.execute("COMMIT TRANSACTION")
preprocess={".pdf":process_pdf, ".odt":process_odt, ".ods":process_ods, ".html":process_striptags, ".xml":process_nothing, ".txt":process_text,
".sql":process_text, ".c":process_text, ".cpp":process_text, ".js":process_text, ".java":process_text,
".py":process_text, '.md':process_text}
def init():
global dbcon
dbcon = sqlite3.connect(config.DBPATH, isolation_level=None)
dbcon = None
if __name__ == '__main__':
with Pool(processes=4,initializer=init) as pool:
if len(sys.argv) < 2:
pool.map(insert, (l.replace("\n", "") for l in sys.stdin))
else:
pool.map(insert, sys.argv[1:])

9
cli/addfileexception.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,9 @@
#ifndef ADDFILEEXCEPTION_H
#define ADDFILEEXCEPTION_H
#include <QException>
#include <QString>
class AddFileException : public QException
{
};
#endif // ADDFILEEXCEPTION_H

46
cli/cli.pro Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,46 @@
QT -= gui
QT += sql concurrent
CONFIG += c++11 console
CONFIG -= app_bundle
# The following define makes your compiler emit warnings if you use
# any feature of Qt which as been marked deprecated (the exact warnings
# depend on your compiler). Please consult the documentation of the
# deprecated API in order to know how to port your code away from it.
DEFINES += QT_DEPRECATED_WARNINGS
# You can also make your code fail to compile if you use deprecated APIs.
# In order to do so, uncomment the following line.
# You can also select to disable deprecated APIs only up to a certain version of Qt.
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0
LIBS += -luchardet -lpoppler-qt5 -lquazip
SOURCES += \
main.cpp \
encodingdetector.cpp \
processor.cpp \
pdfprocessor.cpp \
defaulttextprocessor.cpp \
command.cpp \
commandadd.cpp \
tagstripperprocessor.cpp \
nothingprocessor.cpp \
odtprocessor.cpp \
utils.cpp \
odsprocessor.cpp \
qssgeneralexception.cpp
HEADERS += \
encodingdetector.h \
processor.h \
pagedata.h \
pdfprocessor.h \
defaulttextprocessor.h \
command.h \
commandadd.h \
tagstripperprocessor.h \
nothingprocessor.h \
odtprocessor.h \
utils.h \
odsprocessor.h \
qssgeneralexception.h
INCLUDEPATH += /usr/include/poppler/qt5/ /usr/include/quazip5

37
cli/command.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,37 @@
#include <QFile>
#include <QThread>
#include <QDebug>
#include "command.h"
#include "qssgeneralexception.h"
bool Command::fileExistsInDatabase(QSqlDatabase &db, QString path, qint64 mtime)
{
auto query = QSqlQuery("SELECT 1 FROM file WHERE path = ? and mtime = ?", db);
query.addBindValue(path);
query.addBindValue(mtime);
if(!query.exec())
{ throw QSSGeneralException("Error while trying to query for file existance");
}
if(!query.next())
{
return false;
}
return query.value(0).toBool();
}
QSqlDatabase Command::dbConnection()
{
if(dbStore.hasLocalData())
{
return dbStore.localData();
}
QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE", "QSS" + QString::number((quint64)QThread::currentThread(), 16));
db.setDatabaseName(this->dbConnectionString);
if(!db.open())
{
qDebug() << "Failed to open the database: " << this->dbConnectionString;
}
dbStore.setLocalData(db);
return db;
}

26
cli/command.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,26 @@
#ifndef COMMAND_H
#define COMMAND_H
#include <QStringList>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QThreadStorage>
#include <QVariant>
class Command
{
protected:
bool fileExistsInDatabase(QSqlDatabase &db, QString path, qint64 mtime);
QByteArray readFile(QString path) const;
QString dbConnectionString;
QThreadStorage<QSqlDatabase> dbStore;
public:
Command(QString dbConnectionString)
{
this->dbConnectionString = dbConnectionString;
}
QSqlDatabase dbConnection();
virtual int handle(QStringList arguments) = 0;
virtual ~Command() { };
};
#endif // COMMAND_H

214
cli/commandadd.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,214 @@
#include <QFileInfo>
#include <QDebug>
#include <QSqlQuery>
#include <QSqlError>
#include <QDateTime>
#include <QMap>
#include <QTextStream>
#include <QException>
#include <QCommandLineParser>
#include <QMutex>
#include <QMutexLocker>
#include <QtConcurrent/QtConcurrentMap>
#include "processor.h"
#include "pdfprocessor.h"
#include "commandadd.h"
#include "defaulttextprocessor.h"
#include "tagstripperprocessor.h"
#include "nothingprocessor.h"
#include "odtprocessor.h"
#include "odsprocessor.h"
#include "utils.h"
static DefaultTextProcessor *defaultTextProcessor = new DefaultTextProcessor();
static TagStripperProcessor *tagStripperProcessor = new TagStripperProcessor();
static NothingProcessor *nothingProcessor = new NothingProcessor();
static OdtProcessor *odtProcessor = new OdtProcessor();
static OdsProcessor *odsProcessor = new OdsProcessor();
static QMap<QString, Processor*> processors{
{"pdf", new PdfProcessor() },
{"txt", defaultTextProcessor},
{"md", defaultTextProcessor},
{"py", defaultTextProcessor},
{"xml", nothingProcessor},
{"html", tagStripperProcessor},
{"java", defaultTextProcessor},
{"js", defaultTextProcessor},
{"cpp", defaultTextProcessor},
{"c", defaultTextProcessor},
{"sql", defaultTextProcessor},
{"odt", odtProcessor},
{"ods", odsProcessor}
};
AddFileResult CommandAdd::addFile(QString path)
{
QFileInfo info(path);
QString absPath = info.absoluteFilePath();
auto mtime = info.lastModified().toSecsSinceEpoch();
QChar fileType = info.isDir() ? 'd' : 'f';
QSqlDatabase db = dbConnection();
if(fileExistsInDatabase(db, absPath, mtime))
{
return SKIPPED;
}
Processor *processor = processors.value(info.suffix(), nothingProcessor);
QVector<PageData> pageData;
if(processor->PREFERED_DATA_SOURCE == FILEPATH)
{
pageData = processor->process(absPath);
}
else
{
pageData = processor->process(Utils::readFile(absPath));
}
if(pageData.isEmpty())
{
qDebug() << "Could not get any content for " << absPath;
}
//Workaround to "database is locked" error. Perhaps try WAL mode etc.
//QMutexLocker locker(&writeMutex);
if(!db.transaction())
{
qDebug() << "Failed to open transaction for " << absPath << " : " << db.lastError();
return DBFAIL;
}
QSqlQuery delQuery("DELETE FROM file WHERE path = ?", db);
delQuery.addBindValue(absPath);
if(!delQuery.exec())
{
qDebug() << "Failed DELETE query" << delQuery.lastError();
db.rollback();
return DBFAIL;
}
QSqlQuery inserterQuery("INSERT INTO file(path, mtime, size, filetype) VALUES(?, ?, ?, ?)", db);
inserterQuery.addBindValue(absPath);
inserterQuery.addBindValue(mtime);
inserterQuery.addBindValue(info.size());
inserterQuery.addBindValue(fileType);
if(!inserterQuery.exec())
{
qDebug() << "Failed INSERT query" << inserterQuery.lastError();
db.rollback();
return DBFAIL;
}
int lastid = inserterQuery.lastInsertId().toInt();
for(PageData &data : pageData)
{
QSqlQuery contentQuery("INSERT INTO content(fileid, page, content) VALUES(?, ?, ?)", db);
contentQuery.addBindValue(lastid);
contentQuery.addBindValue(data.pagenumber);
contentQuery.addBindValue(data.content);
if(!contentQuery.exec())
{
db.rollback();
qDebug() << "Failed content insertion " << contentQuery.lastError();
return DBFAIL;
}
}
if(!db.commit())
{
db.rollback();
qDebug() << "Failed to commit transaction for " << absPath << " : " << db.lastError();
return DBFAIL;
}
return OK;
}
int CommandAdd::handle(QStringList arguments)
{
QCommandLineParser parser;
parser.addOptions({
{ { "c", "continue" }, "Continue adding files, don't exit on first error"},
{ { "a", "all" }, "On error, no files should be added, even already processed ones" },
{ { "v", "verbose" }, "Print skipped and added files" },
{ { "n", "threads" }, "Number of threads to use.", "threads" }
});
parser.addHelpOption();
parser.addPositionalArgument("paths", "List of paths to process/add to the index", "[paths...]");
parser.process(arguments);
bool keepGoing = false;
bool verbose = false;
if(parser.isSet("continue"))
{
keepGoing = true;
}
if(parser.isSet("verbose"))
{
verbose = true;
}
if(parser.isSet("all"))
{
throw QSSGeneralException("To be implemented");
}
if(parser.isSet("threads"))
{
QString threadsCount = parser.value("threads");
QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
}
QStringList files = parser.positionalArguments();
if(files.length() == 0)
{
QTextStream stream(stdin);
while(!stream.atEnd())
{
QString path = stream.readLine();
files.append(path);
}
}
bool terminate = false;
QtConcurrent::blockingMap(files, [&](QString &path) {
if(terminate)
{
return;
}
if(verbose)
{
qDebug() << "Processing " << path;
}
auto result = addFile(path);
if(result == DBFAIL)
{
qDebug() << "Failed to add " << path;
if(!keepGoing)
{
terminate = true;
}
}
if(verbose)
{
if(result == SKIPPED)
{
qDebug() << "SKIPPED" << path << "as it already exists in the database";
}
else
{
qDebug() << "Added" << path;
}
}
});
return 0;
}

23
cli/commandadd.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,23 @@
#ifndef COMMANDADD_H
#define COMMANDADD_H
#include <QMutex>
#include "command.h"
enum AddFileResult
{
OK,
SKIPPED,
DBFAIL
};
class CommandAdd : public Command
{
private:
AddFileResult addFile(QString path);
QMutex writeMutex;
public:
using Command::Command;
int handle(QStringList arguments) override;
};
#endif // COMMANDADD_H

31
cli/defaulttextprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,31 @@
#include <QFile>
#include <QDataStream>
#include <QTextCodec>
#include <QDebug>
#include "defaulttextprocessor.h"
DefaultTextProcessor::DefaultTextProcessor()
{
}
QString DefaultTextProcessor::processText(const QByteArray &data) const
{
QString encoding = encodingDetector.detectEncoding(data);
if(!encoding.isEmpty())
{
QTextCodec *codec = QTextCodec::codecForName(encoding.toUtf8());
if(codec != nullptr)
{
return codec->toUnicode(data);
}
qWarning() << "No codec found for " << encoding;
return QString(data);
}
return { };
}
QVector<PageData> DefaultTextProcessor::process(const QByteArray &data) const
{
return {{0, processText(data)}};
}

16
cli/defaulttextprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,16 @@
#ifndef DEFAULTTEXTPROCESSOR_H
#define DEFAULTTEXTPROCESSOR_H
#include "processor.h"
#include "encodingdetector.h"
class DefaultTextProcessor : public Processor
{
protected:
EncodingDetector encodingDetector;
public:
DefaultTextProcessor();
QString processText(const QByteArray &data) const;
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // DEFAULTTEXTPROCESSOR_H

50
cli/encodingdetector.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,50 @@
#include <QDataStream>
#include "encodingdetector.h"
#include <qssgeneralexception.h>
EncodingDetector::EncodingDetector()
{
}
QString EncodingDetector::detectEncoding(const QByteArray &data) const
{
uchardet_t detector = uchardet_new();
if(uchardet_handle_data(detector, data.data(), data.size()) != 0 )
{
uchardet_delete(detector);
throw QSSGeneralException("Decoder failed");
}
uchardet_data_end(detector);
QString encoding = uchardet_get_charset(detector);
uchardet_delete(detector);
return encoding;
}
QString EncodingDetector::detectEncoding(QDataStream &s) const
{
uchardet_t detector = uchardet_new();
char buffer[4096];
int n;
while((n = s.readRawData(buffer, sizeof(buffer))) > 0)
{
if(uchardet_handle_data(detector, buffer, n) != 0)
{
uchardet_delete(detector);
throw QSSGeneralException("Decoder failed");
}
}
if(n == -1)
{
uchardet_delete(detector);
throw QSSGeneralException("Read failed");
}
uchardet_data_end(detector);
QString encoding = uchardet_get_charset(detector);
uchardet_delete(detector);
return encoding;
}

14
cli/encodingdetector.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,14 @@
#ifndef ENCODINGDETECTOR_H
#define ENCODINGDETECTOR_H
#include <QString>
#include <uchardet/uchardet.h>
class EncodingDetector
{
public:
EncodingDetector();
QString detectEncoding(const QByteArray &data) const;
QString detectEncoding(QDataStream &s) const;
};
#endif // ENCODINGDETECTOR_H

76
cli/main.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,76 @@
#include <QCoreApplication>
#include <QFile>
#include <QFileInfo>
#include <QDateTime>
#include <QDataStream>
#include <QDebug>
#include <QProcessEnvironment>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QMap>
#include <QDebug>
#include <functional>
#include <exception>
#include "encodingdetector.h"
#include "pdfprocessor.h"
#include "defaulttextprocessor.h"
#include "command.h"
#include "commandadd.h"
void printUsage(QString argv0)
{
qInfo() << "Usage: " << argv0 << "command";
}
Command *commandFromName(QString name, QString connectionstring)
{
if(name == "add")
{
return new CommandAdd(connectionstring);
}
if(name == "delete")
{
}
if(name == "update")
{
}
if(name == "search")
{
}
return nullptr;
}
int main(int argc, char *argv[])
{
QCoreApplication app(argc, argv);
QStringList args = app.arguments();
QString argv0 = args.takeFirst();
if(args.length() < 1)
{
printUsage(argv0);
exit(1);
}
QString commandName = args.first();
Command *cmd = commandFromName(commandName, QProcessEnvironment::systemEnvironment().value("QSS_PATH"));
if(cmd != nullptr)
{
try
{
return cmd->handle(args);
}
catch(const QSSGeneralException &e)
{
qDebug() << "Exception caught, message: " << e.message;
}
}
else
{
qDebug() << "Unknown command " << commandName;
}
return 1;
}

6
cli/nothingprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,6 @@
#include "nothingprocessor.h"
NothingProcessor::NothingProcessor()
{
}

19
cli/nothingprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,19 @@
#ifndef NOTHINGPROCESSOR_H
#define NOTHINGPROCESSOR_H
#include <QVector>
#include "processor.h"
#include "pagedata.h"
class NothingProcessor : public Processor
{
public:
NothingProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override {
return { };
}
};
#endif // NOTHINGPROCESSOR_H

6
cli/odsprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,6 @@
#include "odsprocessor.h"
OdsProcessor::OdsProcessor()
{
}

10
cli/odsprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,10 @@
#ifndef ODSPROCESSOR_H
#define ODSPROCESSOR_H
#include "odtprocessor.h"
class OdsProcessor : public OdtProcessor
{
public:
OdsProcessor();
};
#endif // ODSPROCESSOR_H

27
cli/odtprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,27 @@
#include <quazip5/quazip.h>
#include <quazip5/quazipfile.h>
#include "odtprocessor.h"
#include "tagstripperprocessor.h"
QVector<PageData> OdtProcessor::process(const QByteArray &data) const
{
throw QSSGeneralException("Not implemented yet");
}
QVector<PageData> OdtProcessor::process(QString path) const
{
QuaZipFile zipFile(path);
zipFile.setFileName("content.xml");
if(!zipFile.open(QIODevice::ReadOnly))
{
throw QSSGeneralException("Error while opening file " + path);
}
QByteArray entireContent = zipFile.readAll();
if(entireContent.isEmpty())
{
throw QSSGeneralException("Error while reading content.xml of " + path);
}
TagStripperProcessor tsp;
return tsp.process(entireContent);
}

18
cli/odtprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,18 @@
#ifndef ODTPROCESSOR_H
#define ODTPROCESSOR_H
#include "processor.h"
class OdtProcessor : public Processor
{
public:
OdtProcessor()
{
this->PREFERED_DATA_SOURCE = FILEPATH;
}
QVector<PageData> process(const QByteArray &data) const override;
QVector<PageData> process(QString path) const override;
};
#endif // ODTPROCESSOR_H

21
cli/pagedata.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,21 @@
#ifndef PAGEDATA_H
#define PAGEDATA_H
#include <QString>
class PageData
{
public:
unsigned int pagenumber = 0;
QString content;
PageData()
{
}
PageData(unsigned int pagenumber, QString content)
{
this->pagenumber = pagenumber;
this->content = content;
}
};
#endif // PAGEDATA_H

34
cli/pdfprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,34 @@
#include <QScopedPointer>
#include <poppler-qt5.h>
#include "pdfprocessor.h"
PdfProcessor::PdfProcessor()
{
}
QVector<PageData> PdfProcessor::process(const QByteArray &data) const
{
QVector<PageData> result;
QScopedPointer<Poppler::Document> doc(Poppler::Document::loadFromData(data));
if(doc.isNull())
{
return { };
}
QRectF entirePage;
auto pagecount = doc->numPages();
QString entire;
entire.reserve(data.size()); //TODO too much
for(auto i = 0; i < pagecount; i++ )
{
QString text =doc->page(i)->text(entirePage);
result.append({static_cast<unsigned int>(i+1),text });
/*TODO: hack, so we can fts search several words over the whole document, not just pages.
* this of course uses more space and should be solved differently.
*/
entire += text;
}
result.append({0, entire});
return result;
}

13
cli/pdfprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,13 @@
#ifndef PDFPROCESSOR_H
#define PDFPROCESSOR_H
#include "processor.h"
class PdfProcessor : public Processor
{
public:
PdfProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // PDFPROCESSOR_H

7
cli/processor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,7 @@
#include "processor.h"
Processor::Processor()
{
}

32
cli/processor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,32 @@
#ifndef PROCESSOR_H
#define PROCESSOR_H
#include <QVector>
#include <QFile>
#include "pagedata.h"
#include "utils.h"
enum DataSource
{
FILEPATH,
ARRAY
};
class Processor
{
public:
/* Indicates the data source the processor performs best with. For example,
* you do not want to read the entire of a compressed archive just to get the content of
* a single file */
DataSource PREFERED_DATA_SOURCE = ARRAY;
Processor();
virtual QVector<PageData> process(const QByteArray &data) const = 0;
virtual QVector<PageData> process(QString path) const
{
return process(Utils::readFile(path));
}
virtual ~Processor() { }
};
#endif // PROCESSOR_H

2
cli/qssgeneralexception.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,2 @@
#include "qssgeneralexception.h"

15
cli/qssgeneralexception.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,15 @@
#ifndef QSSGENERALEXCEPTION_H
#define QSSGENERALEXCEPTION_H
#include <QException>
class QSSGeneralException : public QException
{
public:
QString message;
QSSGeneralException(QString message) { this->message = message; };
void raise() const override { throw *this; }
QSSGeneralException *clone() const override { return new QSSGeneralException(*this); }
};
#endif // QSSGENERALEXCEPTION_H

15
cli/tagstripperprocessor.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,15 @@
#include "tagstripperprocessor.h"
TagStripperProcessor::TagStripperProcessor()
{
}
QVector<PageData> TagStripperProcessor::process(const QByteArray &data) const
{
auto result = DefaultTextProcessor::process(data);
//TODO: does not work properly with <br> and does not deal with entities...
result[0].content.remove(QRegExp("<[^>]*>"));
return result;
}

14
cli/tagstripperprocessor.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,14 @@
#ifndef XMLSTRIPPERPROCESSOR_H
#define XMLSTRIPPERPROCESSOR_H
#include "defaulttextprocessor.h"
class TagStripperProcessor : public DefaultTextProcessor
{
public:
TagStripperProcessor();
public:
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // XMLSTRIPPERPROCESSOR_H

21
cli/utils.cpp Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,21 @@
#include <QDebug>
#include "utils.h"
Utils::Utils()
{
}
QByteArray Utils::readFile(QString path)
{
QFile file(path);
if(!file.open(QIODevice::ReadOnly))
{
throw QSSGeneralException("Failed to open file: " + path);
}
QByteArray data = file.readAll();
if(data.isEmpty() && file.error() != QFileDevice::FileError::NoError)
{
throw QSSGeneralException("Error reading file: " + path + ", Error: " + file.error());
}
return data;
}

15
cli/utils.h Κανονικό αρχείο
Προβολή Αρχείου

@ -0,0 +1,15 @@
#ifndef UTILS_H
#define UTILS_H
#include <QFile>
#include <QString>
#include <QByteArray>
#include "qssgeneralexception.h"
class Utils
{
public:
Utils();
static QByteArray readFile(QString path);
};
#endif // UTILS_H

5
config.py
Προβολή Αρχείου

@ -1,5 +0,0 @@
import os
DBPATH=os.getenv("QSS_PATH")
if DBPATH == None or DBPATH == "":
print("MIssing env var")
exit(1)

21
delindex
Προβολή Αρχείου

@ -1,21 +0,0 @@
#!/bin/sh
TEMPFILE=$(mktemp)
DBFILE="$QSS_PATH"
function todelete()
{
echo "DELETE FROM file WHERE path = '$1';" >> /"$TEMPFILE"
}
echo "BEGIN TRANSACTION;" >> /"$TEMPFILE"
sqlite3 "$DBFILE" "SELECT path FROM file;"| while read line ; do
[ -e "$line" ] || todelete "$line"
done
echo "COMMIT TRANSACTION;" >> /"$TEMPFILE"
sqlite3 "$DBFILE" < /"$TEMPFILE"

21
searchindex
Προβολή Αρχείου

@ -1,21 +0,0 @@
#!/usr/bin/python3
import sqlite3
import sys
import config
dbcon = sqlite3.connect(config.DBPATH, isolation_level=None)
cursor = dbcon.cursor()
if len(sys.argv) < 2:
print("Error: Missing search")
search=sys.argv[1:]
#TODO: machien parseable
for row in cursor.execute("SELECT file.path, content.page FROM file INNER JOIN content ON file.id = content.fileid INNER JOIN content_fts ON content.id = content_fts.ROWID WHERE content_fts.content MATCH ? ORDER By file.mtime ASC", (search)):
print("File:", row[0], "Page: ", row[1])
dbcon.close()