1 Commits

Autor SHA1 Mensaje Fecha
595684c364 Update docs 2023-05-07 19:46:43 +02:00
Se han modificado 29 ficheros con 53 adiciones y 251 borrados

Ver fichero

@ -28,7 +28,7 @@ There is no need to write the long form of filters. There are also booleans avai
The screenshots in this section may occasionally be slightly outdated, but they are usually recent enough to get an overall impression of the current state of the GUI.
## Current status
Latest version: 2023-05-07, v0.9
Latest version: 2022-11-19, v0.8.1
Please keep in mind: looqs is still at an early stage and may exhibit some weirdness and contain bugs.

Ver fichero

@ -46,8 +46,6 @@ packagesExist(quazip1-qt5) {
}
INCLUDEPATH += $$PWD/../shared
INCLUDEPATH += /usr/include/poppler/qt5/
DEPENDPATH += $$PWD/../shared
win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../shared/release/libshared.a

Ver fichero

@ -24,7 +24,7 @@ QSharedPointer<PreviewResult> PreviewGeneratorOdt::generate(RenderConfig config,
throw LooqsGeneralException("Error while reading content.xml of " + documentPath);
}
TagStripperProcessor tsp;
QString content = tsp.process(entireContent).pages.constFirst().content;
QString content = tsp.process(entireContent).constFirst().content;
PreviewGeneratorPlainText plainTextGenerator;
result->setText(plainTextGenerator.generatePreviewText(content, config, info.fileName()));

Ver fichero

@ -24,9 +24,7 @@ QString DefaultTextProcessor::processText(const QByteArray &data) const
return {};
}
DocumentProcessResult DefaultTextProcessor::process(const QByteArray &data) const
QVector<PageData> DefaultTextProcessor::process(const QByteArray &data) const
{
DocumentProcessResult result;
result.pages.append({0, processText(data)});
return result;
return {{0, processText(data)}};
}

Ver fichero

@ -11,7 +11,7 @@ class DefaultTextProcessor : public Processor
public:
DefaultTextProcessor();
QString processText(const QByteArray &data) const;
DocumentProcessResult process(const QByteArray &data) const override;
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // DEFAULTTEXTPROCESSOR_H

Ver fichero

@ -1,31 +0,0 @@
#include "documentoutlineentry.h"
DocumentOutlineEntry::DocumentOutlineEntry()
{
}
QDataStream &operator<<(QDataStream &out, const DocumentOutlineEntry &pd)
{
out << pd.text << pd.type << pd.destinationPage;
out << pd.children.size();
for(const DocumentOutlineEntry &entry : pd.children)
{
out << entry;
}
return out;
}
QDataStream &operator>>(QDataStream &in, DocumentOutlineEntry &pd)
{
in >> pd.text >> pd.type >> pd.destinationPage;
int numChildren;
in >> numChildren;
for(int i = 0; i < numChildren; i++)
{
DocumentOutlineEntry entry;
in >> entry;
pd.children.append(entry);
}
return in;
}

Ver fichero

@ -1,29 +0,0 @@
#ifndef DOCUMENTOUTLINEENTRY_H
#define DOCUMENTOUTLINEENTRY_H
#include <QMetaType>
#include <QDataStream>
#include <QString>
enum OutlineDestinationType
{
OUTLINE_DESTINATION_TYPE_NONE,
OUTLINE_DESTINATION_TYPE_PAGE
/* In the future, links, or #anchors are possible */
};
class DocumentOutlineEntry
{
public:
DocumentOutlineEntry();
QVector<DocumentOutlineEntry> children;
OutlineDestinationType type;
QString text;
unsigned int destinationPage;
};
Q_DECLARE_METATYPE(DocumentOutlineEntry);
QDataStream &operator<<(QDataStream &out, const DocumentOutlineEntry &pd);
QDataStream &operator>>(QDataStream &in, DocumentOutlineEntry &pd);
#endif // DOCUMENTOUTLINEENTRY_H

Ver fichero

@ -1,39 +0,0 @@
#include "documentprocessresult.h"
QDataStream &operator<<(QDataStream &out, const DocumentProcessResult &pd)
{
out << pd.pages.size();
out << pd.outlines.size();
for(const PageData &pd : pd.pages)
{
out << pd;
}
for(const DocumentOutlineEntry &outline : pd.outlines)
{
out << outline;
}
return out;
}
QDataStream &operator>>(QDataStream &in, DocumentProcessResult &pd)
{
int numPages, numOutlines;
in >> numPages;
in >> numOutlines;
for(int i = 0; i < numPages; i++)
{
PageData data;
in >> data;
pd.pages.append(data);
}
for(int i = 0; i < numOutlines; i++)
{
DocumentOutlineEntry outline;
in >> outline;
pd.outlines.append(outline);
}
return in;
}

Ver fichero

@ -1,17 +0,0 @@
#ifndef DOCUMENTPROCESSRESULT_H
#define DOCUMENTPROCESSRESULT_H
#include <pagedata.h>
#include <documentoutlineentry.h>
class DocumentProcessResult
{
public:
QVector<PageData> pages;
QVector<DocumentOutlineEntry> outlines;
};
Q_DECLARE_METATYPE(DocumentProcessResult);
QDataStream &operator<<(QDataStream &out, const DocumentProcessResult &pd);
QDataStream &operator>>(QDataStream &in, DocumentProcessResult &pd);
#endif // DOCUMENTPROCESSRESULT_H

Ver fichero

@ -110,7 +110,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile
SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
{
DocumentProcessResult processResult;
QVector<PageData> pageData;
QString canonicalPath = fileInfo.canonicalFilePath();
int processorReturnCode = -1;
@ -169,10 +169,11 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
* finishes.
*/
QDataStream in(process.readAllStandardOutput());
if(!in.atEnd())
while(!in.atEnd())
{
in >> processResult;
PageData pd;
in >> pd;
pageData.append(pd);
}
processorReturnCode = process.exitCode();
if(processorReturnCode != OK && processorReturnCode != OK_WASEMPTY)
@ -184,7 +185,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
}
}
}
SaveFileResult result = this->dbService->saveFile(fileInfo, processResult, this->fileSaverOptions.metadataOnly);
SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.metadataOnly);
if(result == OK && processorReturnCode == OK_WASEMPTY)
{
return OK_WASEMPTY;

Ver fichero

@ -29,11 +29,6 @@ bool LooqsQuery::hasContentSearch() const
return (this->getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT;
}
bool LooqsQuery::hasOutlineSearch() const
{
return (this->getTokensMask() & FILTER_OUTLINE_CONTAINS) == FILTER_OUTLINE_CONTAINS;
}
bool LooqsQuery::hasPathSearch() const
{
return (this->getTokensMask() & FILTER_PATH) == FILTER_PATH;
@ -294,10 +289,6 @@ LooqsQuery LooqsQuery::build(QString expression, TokenType loneWordsTokenType, b
{
tokenType = FILTER_TAG_ASSIGNED;
}
else if(filtername == "toc" || filtername == "outline")
{
tokenType = FILTER_OUTLINE_CONTAINS;
}
// TODO: given this is not really a "filter", this feels slightly misplaced here
else if(filtername == "sort")
{

Ver fichero

@ -68,7 +68,6 @@ class LooqsQuery
this->limit = limit;
}
bool hasContentSearch() const;
bool hasOutlineSearch() const;
bool hasPathSearch() const;
void addSortCondition(SortCondition sc);

Ver fichero

@ -1,2 +0,0 @@
CREATE TABLE outline(id INTEGER PRIMARY KEY, fileid INTEGER REFERENCES file (id) ON DELETE CASCADE, text varchar(1024), page integer);
CREATE INDEX outline_fileid ON outline (fileid);

Ver fichero

@ -5,6 +5,5 @@
<file>3.sql</file>
<file>4.sql</file>
<file>5.sql</file>
<file>6.sql</file>
</qresource>
</RCC>

Ver fichero

@ -10,7 +10,7 @@ class NothingProcessor : public Processor
NothingProcessor();
public:
DocumentProcessResult process(const QByteArray & /*data*/) const override
QVector<PageData> process(const QByteArray & /*data*/) const override
{
return {};
}

Ver fichero

@ -3,12 +3,12 @@
#include "odtprocessor.h"
#include "tagstripperprocessor.h"
DocumentProcessResult OdtProcessor::process(const QByteArray & /*data*/) const
QVector<PageData> OdtProcessor::process(const QByteArray & /*data*/) const
{
throw LooqsGeneralException("Not implemented yet");
}
DocumentProcessResult OdtProcessor::process(QString path) const
QVector<PageData> OdtProcessor::process(QString path) const
{
QuaZipFile zipFile(path);
zipFile.setFileName("content.xml");

Ver fichero

@ -8,9 +8,9 @@ class OdtProcessor : public Processor
{
this->PREFERED_DATA_SOURCE = FILEPATH;
}
DocumentProcessResult process(const QByteArray &data) const override;
QVector<PageData> process(const QByteArray &data) const override;
DocumentProcessResult process(QString path) const override;
QVector<PageData> process(QString path) const override;
};
#endif // ODTPROCESSOR_H

Ver fichero

@ -5,30 +5,9 @@ PdfProcessor::PdfProcessor()
{
}
QVector<DocumentOutlineEntry> PdfProcessor::createOutline(const QVector<Poppler::OutlineItem> &outlineItems) const
QVector<PageData> PdfProcessor::process(const QByteArray &data) const
{
QVector<DocumentOutlineEntry> result;
for(const Poppler::OutlineItem &outlineItem : outlineItems)
{
DocumentOutlineEntry documentOutlineEntry;
documentOutlineEntry.text = outlineItem.name();
documentOutlineEntry.type = OUTLINE_DESTINATION_TYPE_PAGE;
if(!outlineItem.destination().isNull())
{
documentOutlineEntry.destinationPage = outlineItem.destination()->pageNumber();
}
if(outlineItem.hasChildren())
{
documentOutlineEntry.children = createOutline(outlineItem.children());
}
result.append(documentOutlineEntry);
}
return result;
}
DocumentProcessResult PdfProcessor::process(const QByteArray &data) const
{
DocumentProcessResult result;
QVector<PageData> result;
QScopedPointer<Poppler::Document> doc(Poppler::Document::loadFromData(data));
if(doc.isNull())
{
@ -47,13 +26,12 @@ DocumentProcessResult PdfProcessor::process(const QByteArray &data) const
for(auto i = 0; i < pagecount; i++)
{
QString text = doc->page(i)->text(entirePage);
result.pages.append({static_cast<unsigned int>(i + 1), text});
result.append({static_cast<unsigned int>(i + 1), text});
/*TODO: hack, so we can fts search several words over the whole document, not just pages.
* this of course uses more space and should be solved differently.
*/
entire += text;
}
result.pages.append({0, entire});
result.outlines = createOutline(doc->outline());
result.append({0, entire});
return result;
}

Ver fichero

@ -1,6 +1,5 @@
#ifndef PDFPROCESSOR_H
#define PDFPROCESSOR_H
#include <poppler-qt5.h>
#include "processor.h"
class PdfProcessor : public Processor
{
@ -8,8 +7,7 @@ class PdfProcessor : public Processor
PdfProcessor();
public:
QVector<DocumentOutlineEntry> createOutline(const QVector<Poppler::OutlineItem> &outlineItems) const;
DocumentProcessResult process(const QByteArray &data) const override;
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // PDFPROCESSOR_H

Ver fichero

@ -2,8 +2,8 @@
#define PROCESSOR_H
#include <QVector>
#include <QFile>
#include "pagedata.h"
#include "utils.h"
#include "documentprocessresult.h"
enum DataSource
{
FILEPATH,
@ -18,8 +18,8 @@ class Processor
* a single file */
DataSource PREFERED_DATA_SOURCE = ARRAY;
Processor();
virtual DocumentProcessResult process(const QByteArray &data) const = 0;
virtual DocumentProcessResult process(QString path) const
virtual QVector<PageData> process(const QByteArray &data) const = 0;
virtual QVector<PageData> process(QString path) const
{
return process(Utils::readFile(path));
}

Ver fichero

@ -65,12 +65,18 @@ void SandboxedProcessor::enableSandbox(QString readablePath)
exile_free_policy(policy);
}
void SandboxedProcessor::printResults(const DocumentProcessResult &result)
void SandboxedProcessor::printResults(const QVector<PageData> &pageData)
{
QFile fsstdout;
fsstdout.open(stdout, QIODevice::WriteOnly);
QDataStream stream(&fsstdout);
stream << result;
for(const PageData &data : pageData)
{
stream << data;
// fsstdout.flush();
}
fsstdout.close();
}
@ -96,7 +102,7 @@ SaveFileResult SandboxedProcessor::process()
return OK;
}
DocumentProcessResult processResult;
QVector<PageData> pageData;
QString absPath = fileInfo.absoluteFilePath();
try
@ -105,13 +111,13 @@ SaveFileResult SandboxedProcessor::process()
{
/* Read access to FS needed... doh..*/
enableSandbox(absPath);
processResult = processor->process(absPath);
pageData = processor->process(absPath);
}
else
{
QByteArray data = Utils::readFile(absPath);
enableSandbox();
processResult = processor->process(data);
pageData = processor->process(data);
}
}
catch(LooqsGeneralException &e)
@ -120,6 +126,6 @@ SaveFileResult SandboxedProcessor::process()
return PROCESSFAIL;
}
printResults(processResult);
return processResult.pages.isEmpty() ? OK_WASEMPTY : OK;
printResults(pageData);
return pageData.isEmpty() ? OK_WASEMPTY : OK;
}

Ver fichero

@ -2,7 +2,7 @@
#define SANDBOXEDPROCESSOR_H
#include <QString>
#include <QMimeDatabase>
#include "documentprocessresult.h"
#include "pagedata.h"
#include "savefileresult.h"
class SandboxedProcessor
@ -12,7 +12,7 @@ class SandboxedProcessor
QMimeDatabase mimeDatabase;
void enableSandbox(QString readablePath = "");
void printResults(const DocumentProcessResult &pageData);
void printResults(const QVector<PageData> &pageData);
public:
SandboxedProcessor(QString filepath)

Ver fichero

@ -42,8 +42,6 @@ SOURCES += sqlitesearch.cpp \
dbmigrator.cpp \
defaulttextprocessor.cpp \
dirscanworker.cpp \
documentoutlineentry.cpp \
documentprocessresult.cpp \
encodingdetector.cpp \
filesaver.cpp \
filescanworker.cpp \
@ -74,8 +72,6 @@ HEADERS += sqlitesearch.h \
dbmigrator.h \
defaulttextprocessor.h \
dirscanworker.h \
documentoutlineentry.h \
documentprocessresult.h \
encodingdetector.h \
filedata.h \
filesaver.h \

Ver fichero

@ -253,29 +253,6 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid
return true;
}
bool SqliteDbService::insertOutline(QSqlDatabase &db, int fileid, const QVector<DocumentOutlineEntry> &outlines)
{
QSqlQuery outlineQuery(db);
outlineQuery.prepare("INSERT INTO outline(fileid, text, page) VALUES(?,?,?)");
outlineQuery.addBindValue(fileid);
for(const DocumentOutlineEntry &outline : outlines)
{
outlineQuery.bindValue(1, outline.text.toLower());
outlineQuery.bindValue(2, outline.destinationPage);
if(!outlineQuery.exec())
{
Logger::error() << "Failed outline insertion " << outlineQuery.lastError() << Qt::endl;
return false;
}
if(!insertOutline(db, fileid, outline.children))
{
Logger::error() << "Failed outline insertion (children)) " << outlineQuery.lastError() << Qt::endl;
return false;
}
}
return true;
}
QSqlQuery SqliteDbService::exec(QString querystr, std::initializer_list<QVariant> args)
{
auto query = QSqlQuery(dbFactory->forCurrentThread());
@ -301,7 +278,7 @@ bool SqliteDbService::execBool(QString querystr, std::initializer_list<QVariant>
return query.value(0).toBool();
}
SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, DocumentProcessResult &processResult, bool pathsOnly)
SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly)
{
QString absPath = fileInfo.absoluteFilePath();
auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
@ -346,24 +323,18 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, DocumentProcessResu
if(!pathsOnly)
{
int lastid = inserterQuery.lastInsertId().toInt();
if(!insertToFTS(false, db, lastid, processResult.pages))
if(!insertToFTS(false, db, lastid, pageData))
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
}
if(!insertToFTS(true, db, lastid, processResult.pages))
if(!insertToFTS(true, db, lastid, pageData))
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
}
if(!insertOutline(db, lastid, processResult.outlines))
{
db.rollback();
Logger::error() << "Failed to insert outline data " << Qt::endl;
return DBFAIL;
}
}
if(!db.commit())

Ver fichero

@ -5,7 +5,7 @@
#include "databasefactory.h"
#include "utils.h"
#include "documentprocessresult.h"
#include "pagedata.h"
#include "filedata.h"
#include "../shared/sqlitesearch.h"
#include "../shared/token.h"
@ -22,7 +22,7 @@ class SqliteDbService
public:
SqliteDbService(DatabaseFactory &dbFactory);
SaveFileResult saveFile(QFileInfo fileInfo, DocumentProcessResult &pageData, bool pathsOnly);
SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly);
bool deleteFile(QString path);
bool fileExistsInDatabase(QString path);
@ -42,7 +42,6 @@ class SqliteDbService
QVector<SearchResult> search(const LooqsQuery &query);
std::optional<QChar> queryFileType(QString absPath);
bool insertOutline(QSqlDatabase &db, int fileid, const QVector<DocumentOutlineEntry> &outlines);
};
#endif // SQLITEDBSERVICE_H

Ver fichero

@ -148,11 +148,6 @@ QPair<QString, QVector<QString>> SqliteSearch::createSql(const Token &token)
return {" file.id IN (SELECT fileid FROM filetag WHERE tagid = (SELECT id FROM tag WHERE name = ?)) ",
{value.toLower()}};
}
if(token.type == FILTER_OUTLINE_CONTAINS)
{
return {" outline.text LIKE '%' || ? || '%' ", {value.toLower()}};
}
throw LooqsGeneralException("Unknown token passed (should not happen)");
}
@ -161,7 +156,6 @@ QSqlQuery SqliteSearch::makeSqlQuery(const LooqsQuery &query)
QString whereSql;
QVector<QString> bindValues;
bool isContentSearch = (query.getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT;
bool isOutlineSearch = query.hasOutlineSearch();
if(query.getTokens().isEmpty())
{
throw LooqsGeneralException("Nothing to search for supplied");
@ -206,22 +200,15 @@ QSqlQuery SqliteSearch::makeSqlQuery(const LooqsQuery &query)
}
else
{
QString pageColumn = "'0' as page";
QString joiners = "";
if(isOutlineSearch)
{
pageColumn = "outline.page as page";
joiners = " INNER JOIN outline ON outline.fileid = file.id ";
}
if(sortSql.isEmpty())
{
sortSql = "ORDER BY file.mtime DESC";
}
prepSql = "SELECT DISTINCT file.path AS path, " + pageColumn +
",file.mtime AS mtime, file.size AS size, "
"file.filetype AS filetype FROM file" +
joiners + " WHERE 1=1 AND " + whereSql + " " + sortSql;
prepSql = "SELECT file.path AS path, '0' as page, file.mtime AS mtime, file.size AS size, file.filetype AS "
"filetype FROM file WHERE 1=1 AND " +
whereSql + " " + sortSql;
}
if(query.getLimit() > 0)
{
prepSql += " LIMIT " + QString::number(query.getLimit());
@ -255,7 +242,7 @@ QVector<SearchResult> SqliteSearch::search(const LooqsQuery &query)
throw LooqsGeneralException("SQL Error: " + dbQuery.lastError().text());
}
bool contentSearch = query.hasContentSearch() || query.hasOutlineSearch();
bool contentSearch = query.hasContentSearch();
while(dbQuery.next())
{
SearchResult result;

Ver fichero

@ -4,11 +4,11 @@ TagStripperProcessor::TagStripperProcessor()
{
}
DocumentProcessResult TagStripperProcessor::process(const QByteArray &data) const
QVector<PageData> TagStripperProcessor::process(const QByteArray &data) const
{
auto result = DefaultTextProcessor::process(data);
// TODO: does not work properly with <br> and does not deal with entities...
Q_ASSERT(result.pages.size() > 0);
result.pages[0].content.remove(QRegExp("<[^>]*>"));
result[0].content.remove(QRegExp("<[^>]*>"));
return result;
}

Ver fichero

@ -8,7 +8,7 @@ class TagStripperProcessor : public DefaultTextProcessor
TagStripperProcessor();
public:
DocumentProcessResult process(const QByteArray &data) const override;
QVector<PageData> process(const QByteArray &data) const override;
};
#endif // XMLSTRIPPERPROCESSOR_H

Ver fichero

@ -20,7 +20,6 @@ enum TokenType
FILTER_PATH_ENDS,
FILTER_PATH_STARTS,
FILTER_TAG_ASSIGNED,
FILTER_OUTLINE_CONTAINS,
FILTER_CONTENT = 512, /* Everything below here is content search (except LIMIT) */
FILTER_CONTENT_CONTAINS,
FILTER_CONTENT_PAGE,