Compare commits
	
		
			14 Commits
		
	
	
		
			2b1dc72410
			...
			WIP/outlin
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| 02dd7b64b5 | |||
| 7c30124743 | |||
| 6a8323f2cf | |||
| 763bc47a89 | |||
| 517e62dca2 | |||
| 0f47f581b3 | |||
| 18b18d5103 | |||
| f4eed7a6ef | |||
| 6878f7846a | |||
| b2ae0e488f | |||
| 02a371b81e | |||
| d960570171 | |||
| c5713f5839 | |||
| 8550506517 | 
| @@ -46,6 +46,8 @@ packagesExist(quazip1-qt5) { | |||||||
| } | } | ||||||
|  |  | ||||||
| INCLUDEPATH += $$PWD/../shared | INCLUDEPATH += $$PWD/../shared | ||||||
|  | INCLUDEPATH += /usr/include/poppler/qt5/ | ||||||
|  |  | ||||||
| DEPENDPATH += $$PWD/../shared | DEPENDPATH += $$PWD/../shared | ||||||
|  |  | ||||||
| win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../shared/release/libshared.a | win32-g++:CONFIG(release, debug|release): PRE_TARGETDEPS += $$OUT_PWD/../shared/release/libshared.a | ||||||
|   | |||||||
| @@ -24,7 +24,7 @@ QSharedPointer<PreviewResult> PreviewGeneratorOdt::generate(RenderConfig config, | |||||||
| 		throw LooqsGeneralException("Error while reading content.xml of " + documentPath); | 		throw LooqsGeneralException("Error while reading content.xml of " + documentPath); | ||||||
| 	} | 	} | ||||||
| 	TagStripperProcessor tsp; | 	TagStripperProcessor tsp; | ||||||
| 	QString content = tsp.process(entireContent).constFirst().content; | 	QString content = tsp.process(entireContent).pages.constFirst().content; | ||||||
|  |  | ||||||
| 	PreviewGeneratorPlainText plainTextGenerator; | 	PreviewGeneratorPlainText plainTextGenerator; | ||||||
| 	result->setText(plainTextGenerator.generatePreviewText(content, config, info.fileName())); | 	result->setText(plainTextGenerator.generatePreviewText(content, config, info.fileName())); | ||||||
|   | |||||||
| @@ -24,7 +24,9 @@ QString DefaultTextProcessor::processText(const QByteArray &data) const | |||||||
| 	return {}; | 	return {}; | ||||||
| } | } | ||||||
|  |  | ||||||
| QVector<PageData> DefaultTextProcessor::process(const QByteArray &data) const | DocumentProcessResult DefaultTextProcessor::process(const QByteArray &data) const | ||||||
| { | { | ||||||
| 	return {{0, processText(data)}}; | 	DocumentProcessResult result; | ||||||
|  | 	result.pages.append({0, processText(data)}); | ||||||
|  | 	return result; | ||||||
| } | } | ||||||
|   | |||||||
| @@ -11,7 +11,7 @@ class DefaultTextProcessor : public Processor | |||||||
|   public: |   public: | ||||||
| 	DefaultTextProcessor(); | 	DefaultTextProcessor(); | ||||||
| 	QString processText(const QByteArray &data) const; | 	QString processText(const QByteArray &data) const; | ||||||
| 	QVector<PageData> process(const QByteArray &data) const override; | 	DocumentProcessResult process(const QByteArray &data) const override; | ||||||
| }; | }; | ||||||
|  |  | ||||||
| #endif // DEFAULTTEXTPROCESSOR_H | #endif // DEFAULTTEXTPROCESSOR_H | ||||||
|   | |||||||
							
								
								
									
										31
									
								
								shared/documentoutlineentry.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										31
									
								
								shared/documentoutlineentry.cpp
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,31 @@ | |||||||
|  | #include "documentoutlineentry.h" | ||||||
|  |  | ||||||
|  | DocumentOutlineEntry::DocumentOutlineEntry() | ||||||
|  | { | ||||||
|  | } | ||||||
|  |  | ||||||
|  | QDataStream &operator<<(QDataStream &out, const DocumentOutlineEntry &pd) | ||||||
|  | { | ||||||
|  | 	out << pd.text << pd.type << pd.destinationPage; | ||||||
|  | 	out << pd.children.size(); | ||||||
|  | 	for(const DocumentOutlineEntry &entry : pd.children) | ||||||
|  | 	{ | ||||||
|  | 		out << entry; | ||||||
|  | 	} | ||||||
|  | 	return out; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | QDataStream &operator>>(QDataStream &in, DocumentOutlineEntry &pd) | ||||||
|  | { | ||||||
|  | 	in >> pd.text >> pd.type >> pd.destinationPage; | ||||||
|  |  | ||||||
|  | 	int numChildren; | ||||||
|  | 	in >> numChildren; | ||||||
|  | 	for(int i = 0; i < numChildren; i++) | ||||||
|  | 	{ | ||||||
|  | 		DocumentOutlineEntry entry; | ||||||
|  | 		in >> entry; | ||||||
|  | 		pd.children.append(entry); | ||||||
|  | 	} | ||||||
|  | 	return in; | ||||||
|  | } | ||||||
							
								
								
									
										29
									
								
								shared/documentoutlineentry.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										29
									
								
								shared/documentoutlineentry.h
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,29 @@ | |||||||
|  | #ifndef DOCUMENTOUTLINEENTRY_H | ||||||
|  | #define DOCUMENTOUTLINEENTRY_H | ||||||
|  | #include <QMetaType> | ||||||
|  | #include <QDataStream> | ||||||
|  | #include <QString> | ||||||
|  |  | ||||||
|  | enum OutlineDestinationType | ||||||
|  | { | ||||||
|  | 	OUTLINE_DESTINATION_TYPE_NONE, | ||||||
|  | 	OUTLINE_DESTINATION_TYPE_PAGE | ||||||
|  | 	/* In the future, links, or #anchors are possible */ | ||||||
|  | }; | ||||||
|  |  | ||||||
|  | class DocumentOutlineEntry | ||||||
|  | { | ||||||
|  |   public: | ||||||
|  | 	DocumentOutlineEntry(); | ||||||
|  | 	QVector<DocumentOutlineEntry> children; | ||||||
|  | 	OutlineDestinationType type; | ||||||
|  | 	QString text; | ||||||
|  | 	unsigned int destinationPage; | ||||||
|  | }; | ||||||
|  |  | ||||||
|  | Q_DECLARE_METATYPE(DocumentOutlineEntry); | ||||||
|  |  | ||||||
|  | QDataStream &operator<<(QDataStream &out, const DocumentOutlineEntry &pd); | ||||||
|  | QDataStream &operator>>(QDataStream &in, DocumentOutlineEntry &pd); | ||||||
|  |  | ||||||
|  | #endif // DOCUMENTOUTLINEENTRY_H | ||||||
							
								
								
									
										39
									
								
								shared/documentprocessresult.cpp
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										39
									
								
								shared/documentprocessresult.cpp
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,39 @@ | |||||||
|  | #include "documentprocessresult.h" | ||||||
|  |  | ||||||
|  | QDataStream &operator<<(QDataStream &out, const DocumentProcessResult &pd) | ||||||
|  | { | ||||||
|  | 	out << pd.pages.size(); | ||||||
|  | 	out << pd.outlines.size(); | ||||||
|  | 	for(const PageData &pd : pd.pages) | ||||||
|  | 	{ | ||||||
|  | 		out << pd; | ||||||
|  | 	} | ||||||
|  | 	for(const DocumentOutlineEntry &outline : pd.outlines) | ||||||
|  | 	{ | ||||||
|  | 		out << outline; | ||||||
|  | 	} | ||||||
|  | 	return out; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | QDataStream &operator>>(QDataStream &in, DocumentProcessResult &pd) | ||||||
|  | { | ||||||
|  | 	int numPages, numOutlines; | ||||||
|  | 	in >> numPages; | ||||||
|  | 	in >> numOutlines; | ||||||
|  |  | ||||||
|  | 	for(int i = 0; i < numPages; i++) | ||||||
|  | 	{ | ||||||
|  | 		PageData data; | ||||||
|  | 		in >> data; | ||||||
|  | 		pd.pages.append(data); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	for(int i = 0; i < numOutlines; i++) | ||||||
|  | 	{ | ||||||
|  | 		DocumentOutlineEntry outline; | ||||||
|  | 		in >> outline; | ||||||
|  | 		pd.outlines.append(outline); | ||||||
|  | 	} | ||||||
|  |  | ||||||
|  | 	return in; | ||||||
|  | } | ||||||
							
								
								
									
										17
									
								
								shared/documentprocessresult.h
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										17
									
								
								shared/documentprocessresult.h
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,17 @@ | |||||||
|  | #ifndef DOCUMENTPROCESSRESULT_H | ||||||
|  | #define DOCUMENTPROCESSRESULT_H | ||||||
|  | #include <pagedata.h> | ||||||
|  | #include <documentoutlineentry.h> | ||||||
|  |  | ||||||
|  | class DocumentProcessResult | ||||||
|  | { | ||||||
|  |   public: | ||||||
|  | 	QVector<PageData> pages; | ||||||
|  | 	QVector<DocumentOutlineEntry> outlines; | ||||||
|  | }; | ||||||
|  | Q_DECLARE_METATYPE(DocumentProcessResult); | ||||||
|  |  | ||||||
|  | QDataStream &operator<<(QDataStream &out, const DocumentProcessResult &pd); | ||||||
|  | QDataStream &operator>>(QDataStream &in, DocumentProcessResult &pd); | ||||||
|  |  | ||||||
|  | #endif // DOCUMENTPROCESSRESULT_H | ||||||
| @@ -110,7 +110,7 @@ int FileSaver::processFiles(const QVector<QString> paths, std::function<SaveFile | |||||||
|  |  | ||||||
| SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) | SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) | ||||||
| { | { | ||||||
| 	QVector<PageData> pageData; | 	DocumentProcessResult processResult; | ||||||
| 	QString canonicalPath = fileInfo.canonicalFilePath(); | 	QString canonicalPath = fileInfo.canonicalFilePath(); | ||||||
|  |  | ||||||
| 	int processorReturnCode = -1; | 	int processorReturnCode = -1; | ||||||
| @@ -169,11 +169,10 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) | |||||||
| 			 * finishes. | 			 * finishes. | ||||||
| 			 */ | 			 */ | ||||||
| 			QDataStream in(process.readAllStandardOutput()); | 			QDataStream in(process.readAllStandardOutput()); | ||||||
| 			while(!in.atEnd()) |  | ||||||
|  | 			if(!in.atEnd()) | ||||||
| 			{ | 			{ | ||||||
| 				PageData pd; | 				in >> processResult; | ||||||
| 				in >> pd; |  | ||||||
| 				pageData.append(pd); |  | ||||||
| 			} | 			} | ||||||
| 			processorReturnCode = process.exitCode(); | 			processorReturnCode = process.exitCode(); | ||||||
| 			if(processorReturnCode != OK && processorReturnCode != OK_WASEMPTY) | 			if(processorReturnCode != OK && processorReturnCode != OK_WASEMPTY) | ||||||
| @@ -185,7 +184,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo) | |||||||
| 			} | 			} | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| 	SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.metadataOnly); | 	SaveFileResult result = this->dbService->saveFile(fileInfo, processResult, this->fileSaverOptions.metadataOnly); | ||||||
| 	if(result == OK && processorReturnCode == OK_WASEMPTY) | 	if(result == OK && processorReturnCode == OK_WASEMPTY) | ||||||
| 	{ | 	{ | ||||||
| 		return OK_WASEMPTY; | 		return OK_WASEMPTY; | ||||||
|   | |||||||
| @@ -29,6 +29,11 @@ bool LooqsQuery::hasContentSearch() const | |||||||
| 	return (this->getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT; | 	return (this->getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT; | ||||||
| } | } | ||||||
|  |  | ||||||
|  | bool LooqsQuery::hasOutlineSearch() const | ||||||
|  | { | ||||||
|  | 	return (this->getTokensMask() & FILTER_OUTLINE_CONTAINS) == FILTER_OUTLINE_CONTAINS; | ||||||
|  | } | ||||||
|  |  | ||||||
| bool LooqsQuery::hasPathSearch() const | bool LooqsQuery::hasPathSearch() const | ||||||
| { | { | ||||||
| 	return (this->getTokensMask() & FILTER_PATH) == FILTER_PATH; | 	return (this->getTokensMask() & FILTER_PATH) == FILTER_PATH; | ||||||
| @@ -289,6 +294,10 @@ LooqsQuery LooqsQuery::build(QString expression, TokenType loneWordsTokenType, b | |||||||
| 			{ | 			{ | ||||||
| 				tokenType = FILTER_TAG_ASSIGNED; | 				tokenType = FILTER_TAG_ASSIGNED; | ||||||
| 			} | 			} | ||||||
|  | 			else if(filtername == "toc" || filtername == "outline") | ||||||
|  | 			{ | ||||||
|  | 				tokenType = FILTER_OUTLINE_CONTAINS; | ||||||
|  | 			} | ||||||
| 			// TODO: given this is not really a "filter", this feels slightly misplaced here | 			// TODO: given this is not really a "filter", this feels slightly misplaced here | ||||||
| 			else if(filtername == "sort") | 			else if(filtername == "sort") | ||||||
| 			{ | 			{ | ||||||
|   | |||||||
| @@ -68,6 +68,7 @@ class LooqsQuery | |||||||
| 		this->limit = limit; | 		this->limit = limit; | ||||||
| 	} | 	} | ||||||
| 	bool hasContentSearch() const; | 	bool hasContentSearch() const; | ||||||
|  | 	bool hasOutlineSearch() const; | ||||||
| 	bool hasPathSearch() const; | 	bool hasPathSearch() const; | ||||||
|  |  | ||||||
| 	void addSortCondition(SortCondition sc); | 	void addSortCondition(SortCondition sc); | ||||||
|   | |||||||
							
								
								
									
										2
									
								
								shared/migrations/6.sql
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										2
									
								
								shared/migrations/6.sql
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,2 @@ | |||||||
|  | CREATE TABLE outline(id INTEGER PRIMARY KEY, fileid INTEGER REFERENCES file (id) ON DELETE CASCADE, text varchar(1024), page integer); | ||||||
|  | CREATE INDEX outline_fileid ON outline (fileid); | ||||||
| @@ -5,5 +5,6 @@ | |||||||
|         <file>3.sql</file> |         <file>3.sql</file> | ||||||
|         <file>4.sql</file> |         <file>4.sql</file> | ||||||
|         <file>5.sql</file> |         <file>5.sql</file> | ||||||
|  |         <file>6.sql</file> | ||||||
|     </qresource> |     </qresource> | ||||||
| </RCC> | </RCC> | ||||||
|   | |||||||
| @@ -10,7 +10,7 @@ class NothingProcessor : public Processor | |||||||
| 	NothingProcessor(); | 	NothingProcessor(); | ||||||
|  |  | ||||||
|   public: |   public: | ||||||
| 	QVector<PageData> process(const QByteArray & /*data*/) const override | 	DocumentProcessResult process(const QByteArray & /*data*/) const override | ||||||
| 	{ | 	{ | ||||||
| 		return {}; | 		return {}; | ||||||
| 	} | 	} | ||||||
|   | |||||||
| @@ -3,12 +3,12 @@ | |||||||
| #include "odtprocessor.h" | #include "odtprocessor.h" | ||||||
| #include "tagstripperprocessor.h" | #include "tagstripperprocessor.h" | ||||||
|  |  | ||||||
| QVector<PageData> OdtProcessor::process(const QByteArray & /*data*/) const | DocumentProcessResult OdtProcessor::process(const QByteArray & /*data*/) const | ||||||
| { | { | ||||||
| 	throw LooqsGeneralException("Not implemented yet"); | 	throw LooqsGeneralException("Not implemented yet"); | ||||||
| } | } | ||||||
|  |  | ||||||
| QVector<PageData> OdtProcessor::process(QString path) const | DocumentProcessResult OdtProcessor::process(QString path) const | ||||||
| { | { | ||||||
| 	QuaZipFile zipFile(path); | 	QuaZipFile zipFile(path); | ||||||
| 	zipFile.setFileName("content.xml"); | 	zipFile.setFileName("content.xml"); | ||||||
|   | |||||||
| @@ -8,9 +8,9 @@ class OdtProcessor : public Processor | |||||||
| 	{ | 	{ | ||||||
| 		this->PREFERED_DATA_SOURCE = FILEPATH; | 		this->PREFERED_DATA_SOURCE = FILEPATH; | ||||||
| 	} | 	} | ||||||
| 	QVector<PageData> process(const QByteArray &data) const override; | 	DocumentProcessResult process(const QByteArray &data) const override; | ||||||
|  |  | ||||||
| 	QVector<PageData> process(QString path) const override; | 	DocumentProcessResult process(QString path) const override; | ||||||
| }; | }; | ||||||
|  |  | ||||||
| #endif // ODTPROCESSOR_H | #endif // ODTPROCESSOR_H | ||||||
|   | |||||||
| @@ -5,9 +5,30 @@ PdfProcessor::PdfProcessor() | |||||||
| { | { | ||||||
| } | } | ||||||
|  |  | ||||||
| QVector<PageData> PdfProcessor::process(const QByteArray &data) const | QVector<DocumentOutlineEntry> PdfProcessor::createOutline(const QVector<Poppler::OutlineItem> &outlineItems) const | ||||||
| { | { | ||||||
| 	QVector<PageData> result; | 	QVector<DocumentOutlineEntry> result; | ||||||
|  | 	for(const Poppler::OutlineItem &outlineItem : outlineItems) | ||||||
|  | 	{ | ||||||
|  | 		DocumentOutlineEntry documentOutlineEntry; | ||||||
|  | 		documentOutlineEntry.text = outlineItem.name(); | ||||||
|  | 		documentOutlineEntry.type = OUTLINE_DESTINATION_TYPE_PAGE; | ||||||
|  | 		if(!outlineItem.destination().isNull()) | ||||||
|  | 		{ | ||||||
|  | 			documentOutlineEntry.destinationPage = outlineItem.destination()->pageNumber(); | ||||||
|  | 		} | ||||||
|  | 		if(outlineItem.hasChildren()) | ||||||
|  | 		{ | ||||||
|  | 			documentOutlineEntry.children = createOutline(outlineItem.children()); | ||||||
|  | 		} | ||||||
|  | 		result.append(documentOutlineEntry); | ||||||
|  | 	} | ||||||
|  | 	return result; | ||||||
|  | } | ||||||
|  |  | ||||||
|  | DocumentProcessResult PdfProcessor::process(const QByteArray &data) const | ||||||
|  | { | ||||||
|  | 	DocumentProcessResult result; | ||||||
| 	QScopedPointer<Poppler::Document> doc(Poppler::Document::loadFromData(data)); | 	QScopedPointer<Poppler::Document> doc(Poppler::Document::loadFromData(data)); | ||||||
| 	if(doc.isNull()) | 	if(doc.isNull()) | ||||||
| 	{ | 	{ | ||||||
| @@ -26,12 +47,13 @@ QVector<PageData> PdfProcessor::process(const QByteArray &data) const | |||||||
| 	for(auto i = 0; i < pagecount; i++) | 	for(auto i = 0; i < pagecount; i++) | ||||||
| 	{ | 	{ | ||||||
| 		QString text = doc->page(i)->text(entirePage); | 		QString text = doc->page(i)->text(entirePage); | ||||||
| 		result.append({static_cast<unsigned int>(i + 1), text}); | 		result.pages.append({static_cast<unsigned int>(i + 1), text}); | ||||||
| 		/*TODO: hack, so we can fts search several words over the whole document, not just pages. | 		/*TODO: hack, so we can fts search several words over the whole document, not just pages. | ||||||
| 		 * this of course uses more space and should be solved differently. | 		 * this of course uses more space and should be solved differently. | ||||||
| 		 */ | 		 */ | ||||||
| 		entire += text; | 		entire += text; | ||||||
| 	} | 	} | ||||||
| 	result.append({0, entire}); | 	result.pages.append({0, entire}); | ||||||
|  | 	result.outlines = createOutline(doc->outline()); | ||||||
| 	return result; | 	return result; | ||||||
| } | } | ||||||
|   | |||||||
| @@ -1,5 +1,6 @@ | |||||||
| #ifndef PDFPROCESSOR_H | #ifndef PDFPROCESSOR_H | ||||||
| #define PDFPROCESSOR_H | #define PDFPROCESSOR_H | ||||||
|  | #include <poppler-qt5.h> | ||||||
| #include "processor.h" | #include "processor.h" | ||||||
| class PdfProcessor : public Processor | class PdfProcessor : public Processor | ||||||
| { | { | ||||||
| @@ -7,7 +8,8 @@ class PdfProcessor : public Processor | |||||||
| 	PdfProcessor(); | 	PdfProcessor(); | ||||||
|  |  | ||||||
|   public: |   public: | ||||||
| 	QVector<PageData> process(const QByteArray &data) const override; | 	QVector<DocumentOutlineEntry> createOutline(const QVector<Poppler::OutlineItem> &outlineItems) const; | ||||||
|  | 	DocumentProcessResult process(const QByteArray &data) const override; | ||||||
| }; | }; | ||||||
|  |  | ||||||
| #endif // PDFPROCESSOR_H | #endif // PDFPROCESSOR_H | ||||||
|   | |||||||
| @@ -2,8 +2,8 @@ | |||||||
| #define PROCESSOR_H | #define PROCESSOR_H | ||||||
| #include <QVector> | #include <QVector> | ||||||
| #include <QFile> | #include <QFile> | ||||||
| #include "pagedata.h" |  | ||||||
| #include "utils.h" | #include "utils.h" | ||||||
|  | #include "documentprocessresult.h" | ||||||
| enum DataSource | enum DataSource | ||||||
| { | { | ||||||
| 	FILEPATH, | 	FILEPATH, | ||||||
| @@ -18,8 +18,8 @@ class Processor | |||||||
| 	 * a single file */ | 	 * a single file */ | ||||||
| 	DataSource PREFERED_DATA_SOURCE = ARRAY; | 	DataSource PREFERED_DATA_SOURCE = ARRAY; | ||||||
| 	Processor(); | 	Processor(); | ||||||
| 	virtual QVector<PageData> process(const QByteArray &data) const = 0; | 	virtual DocumentProcessResult process(const QByteArray &data) const = 0; | ||||||
| 	virtual QVector<PageData> process(QString path) const | 	virtual DocumentProcessResult process(QString path) const | ||||||
| 	{ | 	{ | ||||||
| 		return process(Utils::readFile(path)); | 		return process(Utils::readFile(path)); | ||||||
| 	} | 	} | ||||||
|   | |||||||
| @@ -65,18 +65,12 @@ void SandboxedProcessor::enableSandbox(QString readablePath) | |||||||
| 	exile_free_policy(policy); | 	exile_free_policy(policy); | ||||||
| } | } | ||||||
|  |  | ||||||
| void SandboxedProcessor::printResults(const QVector<PageData> &pageData) | void SandboxedProcessor::printResults(const DocumentProcessResult &result) | ||||||
| { | { | ||||||
| 	QFile fsstdout; | 	QFile fsstdout; | ||||||
| 	fsstdout.open(stdout, QIODevice::WriteOnly); | 	fsstdout.open(stdout, QIODevice::WriteOnly); | ||||||
| 	QDataStream stream(&fsstdout); | 	QDataStream stream(&fsstdout); | ||||||
|  | 	stream << result; | ||||||
| 	for(const PageData &data : pageData) |  | ||||||
| 	{ |  | ||||||
| 		stream << data; |  | ||||||
| 		// fsstdout.flush(); |  | ||||||
| 	} |  | ||||||
|  |  | ||||||
| 	fsstdout.close(); | 	fsstdout.close(); | ||||||
| } | } | ||||||
|  |  | ||||||
| @@ -102,7 +96,7 @@ SaveFileResult SandboxedProcessor::process() | |||||||
| 		return OK; | 		return OK; | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	QVector<PageData> pageData; | 	DocumentProcessResult processResult; | ||||||
| 	QString absPath = fileInfo.absoluteFilePath(); | 	QString absPath = fileInfo.absoluteFilePath(); | ||||||
|  |  | ||||||
| 	try | 	try | ||||||
| @@ -111,13 +105,13 @@ SaveFileResult SandboxedProcessor::process() | |||||||
| 		{ | 		{ | ||||||
| 			/* Read access to FS needed... doh..*/ | 			/* Read access to FS needed... doh..*/ | ||||||
| 			enableSandbox(absPath); | 			enableSandbox(absPath); | ||||||
| 			pageData = processor->process(absPath); | 			processResult = processor->process(absPath); | ||||||
| 		} | 		} | ||||||
| 		else | 		else | ||||||
| 		{ | 		{ | ||||||
| 			QByteArray data = Utils::readFile(absPath); | 			QByteArray data = Utils::readFile(absPath); | ||||||
| 			enableSandbox(); | 			enableSandbox(); | ||||||
| 			pageData = processor->process(data); | 			processResult = processor->process(data); | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| 	catch(LooqsGeneralException &e) | 	catch(LooqsGeneralException &e) | ||||||
| @@ -126,6 +120,6 @@ SaveFileResult SandboxedProcessor::process() | |||||||
| 		return PROCESSFAIL; | 		return PROCESSFAIL; | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	printResults(pageData); | 	printResults(processResult); | ||||||
| 	return pageData.isEmpty() ? OK_WASEMPTY : OK; | 	return processResult.pages.isEmpty() ? OK_WASEMPTY : OK; | ||||||
| } | } | ||||||
|   | |||||||
| @@ -2,7 +2,7 @@ | |||||||
| #define SANDBOXEDPROCESSOR_H | #define SANDBOXEDPROCESSOR_H | ||||||
| #include <QString> | #include <QString> | ||||||
| #include <QMimeDatabase> | #include <QMimeDatabase> | ||||||
| #include "pagedata.h" | #include "documentprocessresult.h" | ||||||
| #include "savefileresult.h" | #include "savefileresult.h" | ||||||
|  |  | ||||||
| class SandboxedProcessor | class SandboxedProcessor | ||||||
| @@ -12,7 +12,7 @@ class SandboxedProcessor | |||||||
| 	QMimeDatabase mimeDatabase; | 	QMimeDatabase mimeDatabase; | ||||||
|  |  | ||||||
| 	void enableSandbox(QString readablePath = ""); | 	void enableSandbox(QString readablePath = ""); | ||||||
| 	void printResults(const QVector<PageData> &pageData); | 	void printResults(const DocumentProcessResult &pageData); | ||||||
|  |  | ||||||
|   public: |   public: | ||||||
| 	SandboxedProcessor(QString filepath) | 	SandboxedProcessor(QString filepath) | ||||||
|   | |||||||
| @@ -42,6 +42,8 @@ SOURCES += sqlitesearch.cpp \ | |||||||
|     dbmigrator.cpp \ |     dbmigrator.cpp \ | ||||||
|     defaulttextprocessor.cpp \ |     defaulttextprocessor.cpp \ | ||||||
|     dirscanworker.cpp \ |     dirscanworker.cpp \ | ||||||
|  |     documentoutlineentry.cpp \ | ||||||
|  |     documentprocessresult.cpp \ | ||||||
|     encodingdetector.cpp \ |     encodingdetector.cpp \ | ||||||
|     filesaver.cpp \ |     filesaver.cpp \ | ||||||
|     filescanworker.cpp \ |     filescanworker.cpp \ | ||||||
| @@ -72,6 +74,8 @@ HEADERS += sqlitesearch.h \ | |||||||
|     dbmigrator.h \ |     dbmigrator.h \ | ||||||
|     defaulttextprocessor.h \ |     defaulttextprocessor.h \ | ||||||
|     dirscanworker.h \ |     dirscanworker.h \ | ||||||
|  |     documentoutlineentry.h \ | ||||||
|  |     documentprocessresult.h \ | ||||||
|     encodingdetector.h \ |     encodingdetector.h \ | ||||||
|     filedata.h \ |     filedata.h \ | ||||||
|     filesaver.h \ |     filesaver.h \ | ||||||
|   | |||||||
| @@ -253,6 +253,29 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid | |||||||
| 	return true; | 	return true; | ||||||
| } | } | ||||||
|  |  | ||||||
|  | bool SqliteDbService::insertOutline(QSqlDatabase &db, int fileid, const QVector<DocumentOutlineEntry> &outlines) | ||||||
|  | { | ||||||
|  | 	QSqlQuery outlineQuery(db); | ||||||
|  | 	outlineQuery.prepare("INSERT INTO outline(fileid, text, page) VALUES(?,?,?)"); | ||||||
|  | 	outlineQuery.addBindValue(fileid); | ||||||
|  | 	for(const DocumentOutlineEntry &outline : outlines) | ||||||
|  | 	{ | ||||||
|  | 		outlineQuery.bindValue(1, outline.text.toLower()); | ||||||
|  | 		outlineQuery.bindValue(2, outline.destinationPage); | ||||||
|  | 		if(!outlineQuery.exec()) | ||||||
|  | 		{ | ||||||
|  | 			Logger::error() << "Failed outline insertion " << outlineQuery.lastError() << Qt::endl; | ||||||
|  | 			return false; | ||||||
|  | 		} | ||||||
|  | 		if(!insertOutline(db, fileid, outline.children)) | ||||||
|  | 		{ | ||||||
|  | 			Logger::error() << "Failed outline insertion (children)) " << outlineQuery.lastError() << Qt::endl; | ||||||
|  | 			return false; | ||||||
|  | 		} | ||||||
|  | 	} | ||||||
|  | 	return true; | ||||||
|  | } | ||||||
|  |  | ||||||
| QSqlQuery SqliteDbService::exec(QString querystr, std::initializer_list<QVariant> args) | QSqlQuery SqliteDbService::exec(QString querystr, std::initializer_list<QVariant> args) | ||||||
| { | { | ||||||
| 	auto query = QSqlQuery(dbFactory->forCurrentThread()); | 	auto query = QSqlQuery(dbFactory->forCurrentThread()); | ||||||
| @@ -278,7 +301,7 @@ bool SqliteDbService::execBool(QString querystr, std::initializer_list<QVariant> | |||||||
| 	return query.value(0).toBool(); | 	return query.value(0).toBool(); | ||||||
| } | } | ||||||
|  |  | ||||||
| SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly) | SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, DocumentProcessResult &processResult, bool pathsOnly) | ||||||
| { | { | ||||||
| 	QString absPath = fileInfo.absoluteFilePath(); | 	QString absPath = fileInfo.absoluteFilePath(); | ||||||
| 	auto mtime = fileInfo.lastModified().toSecsSinceEpoch(); | 	auto mtime = fileInfo.lastModified().toSecsSinceEpoch(); | ||||||
| @@ -323,18 +346,24 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> & | |||||||
| 	if(!pathsOnly) | 	if(!pathsOnly) | ||||||
| 	{ | 	{ | ||||||
| 		int lastid = inserterQuery.lastInsertId().toInt(); | 		int lastid = inserterQuery.lastInsertId().toInt(); | ||||||
| 		if(!insertToFTS(false, db, lastid, pageData)) | 		if(!insertToFTS(false, db, lastid, processResult.pages)) | ||||||
| 		{ | 		{ | ||||||
| 			db.rollback(); | 			db.rollback(); | ||||||
| 			Logger::error() << "Failed to insert data to FTS index " << Qt::endl; | 			Logger::error() << "Failed to insert data to FTS index " << Qt::endl; | ||||||
| 			return DBFAIL; | 			return DBFAIL; | ||||||
| 		} | 		} | ||||||
| 		if(!insertToFTS(true, db, lastid, pageData)) | 		if(!insertToFTS(true, db, lastid, processResult.pages)) | ||||||
| 		{ | 		{ | ||||||
| 			db.rollback(); | 			db.rollback(); | ||||||
| 			Logger::error() << "Failed to insert data to FTS index " << Qt::endl; | 			Logger::error() << "Failed to insert data to FTS index " << Qt::endl; | ||||||
| 			return DBFAIL; | 			return DBFAIL; | ||||||
| 		} | 		} | ||||||
|  | 		if(!insertOutline(db, lastid, processResult.outlines)) | ||||||
|  | 		{ | ||||||
|  | 			db.rollback(); | ||||||
|  | 			Logger::error() << "Failed to insert outline data " << Qt::endl; | ||||||
|  | 			return DBFAIL; | ||||||
|  | 		} | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	if(!db.commit()) | 	if(!db.commit()) | ||||||
|   | |||||||
| @@ -5,7 +5,7 @@ | |||||||
|  |  | ||||||
| #include "databasefactory.h" | #include "databasefactory.h" | ||||||
| #include "utils.h" | #include "utils.h" | ||||||
| #include "pagedata.h" | #include "documentprocessresult.h" | ||||||
| #include "filedata.h" | #include "filedata.h" | ||||||
| #include "../shared/sqlitesearch.h" | #include "../shared/sqlitesearch.h" | ||||||
| #include "../shared/token.h" | #include "../shared/token.h" | ||||||
| @@ -22,7 +22,7 @@ class SqliteDbService | |||||||
|  |  | ||||||
|   public: |   public: | ||||||
| 	SqliteDbService(DatabaseFactory &dbFactory); | 	SqliteDbService(DatabaseFactory &dbFactory); | ||||||
| 	SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly); | 	SaveFileResult saveFile(QFileInfo fileInfo, DocumentProcessResult &pageData, bool pathsOnly); | ||||||
|  |  | ||||||
| 	bool deleteFile(QString path); | 	bool deleteFile(QString path); | ||||||
| 	bool fileExistsInDatabase(QString path); | 	bool fileExistsInDatabase(QString path); | ||||||
| @@ -42,6 +42,7 @@ class SqliteDbService | |||||||
| 	QVector<SearchResult> search(const LooqsQuery &query); | 	QVector<SearchResult> search(const LooqsQuery &query); | ||||||
|  |  | ||||||
| 	std::optional<QChar> queryFileType(QString absPath); | 	std::optional<QChar> queryFileType(QString absPath); | ||||||
|  | 	bool insertOutline(QSqlDatabase &db, int fileid, const QVector<DocumentOutlineEntry> &outlines); | ||||||
| }; | }; | ||||||
|  |  | ||||||
| #endif // SQLITEDBSERVICE_H | #endif // SQLITEDBSERVICE_H | ||||||
|   | |||||||
| @@ -148,6 +148,11 @@ QPair<QString, QVector<QString>> SqliteSearch::createSql(const Token &token) | |||||||
| 		return {" file.id IN (SELECT fileid FROM filetag WHERE tagid = (SELECT id FROM tag WHERE name = ?)) ", | 		return {" file.id IN (SELECT fileid FROM filetag WHERE tagid = (SELECT id FROM tag WHERE name = ?)) ", | ||||||
| 				{value.toLower()}}; | 				{value.toLower()}}; | ||||||
| 	} | 	} | ||||||
|  | 	if(token.type == FILTER_OUTLINE_CONTAINS) | ||||||
|  | 	{ | ||||||
|  | 		return {" outline.text LIKE '%' || ? || '%'  ", {value.toLower()}}; | ||||||
|  | 	} | ||||||
|  |  | ||||||
| 	throw LooqsGeneralException("Unknown token passed (should not happen)"); | 	throw LooqsGeneralException("Unknown token passed (should not happen)"); | ||||||
| } | } | ||||||
|  |  | ||||||
| @@ -156,6 +161,7 @@ QSqlQuery SqliteSearch::makeSqlQuery(const LooqsQuery &query) | |||||||
| 	QString whereSql; | 	QString whereSql; | ||||||
| 	QVector<QString> bindValues; | 	QVector<QString> bindValues; | ||||||
| 	bool isContentSearch = (query.getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT; | 	bool isContentSearch = (query.getTokensMask() & FILTER_CONTENT) == FILTER_CONTENT; | ||||||
|  | 	bool isOutlineSearch = query.hasOutlineSearch(); | ||||||
| 	if(query.getTokens().isEmpty()) | 	if(query.getTokens().isEmpty()) | ||||||
| 	{ | 	{ | ||||||
| 		throw LooqsGeneralException("Nothing to search for supplied"); | 		throw LooqsGeneralException("Nothing to search for supplied"); | ||||||
| @@ -200,15 +206,22 @@ QSqlQuery SqliteSearch::makeSqlQuery(const LooqsQuery &query) | |||||||
| 	} | 	} | ||||||
| 	else | 	else | ||||||
| 	{ | 	{ | ||||||
|  | 		QString pageColumn = "'0' as page"; | ||||||
|  | 		QString joiners = ""; | ||||||
|  | 		if(isOutlineSearch) | ||||||
|  | 		{ | ||||||
|  | 			pageColumn = "outline.page as page"; | ||||||
|  | 			joiners = " INNER JOIN outline ON outline.fileid = file.id "; | ||||||
|  | 		} | ||||||
| 		if(sortSql.isEmpty()) | 		if(sortSql.isEmpty()) | ||||||
| 		{ | 		{ | ||||||
| 			sortSql = "ORDER BY file.mtime DESC"; | 			sortSql = "ORDER BY file.mtime DESC"; | ||||||
| 		} | 		} | ||||||
| 		prepSql = "SELECT file.path AS path, '0' as page,  file.mtime AS mtime, file.size AS size, file.filetype AS " | 		prepSql = "SELECT DISTINCT file.path AS path, " + pageColumn + | ||||||
| 				  "filetype FROM file WHERE  1=1 AND " + | 				  ",file.mtime AS mtime, file.size AS size, " | ||||||
| 				  whereSql + " " + sortSql; | 				  "file.filetype AS filetype FROM file" + | ||||||
|  | 				  joiners + " WHERE  1=1 AND " + whereSql + " " + sortSql; | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	if(query.getLimit() > 0) | 	if(query.getLimit() > 0) | ||||||
| 	{ | 	{ | ||||||
| 		prepSql += " LIMIT " + QString::number(query.getLimit()); | 		prepSql += " LIMIT " + QString::number(query.getLimit()); | ||||||
| @@ -242,7 +255,7 @@ QVector<SearchResult> SqliteSearch::search(const LooqsQuery &query) | |||||||
| 		throw LooqsGeneralException("SQL Error: " + dbQuery.lastError().text()); | 		throw LooqsGeneralException("SQL Error: " + dbQuery.lastError().text()); | ||||||
| 	} | 	} | ||||||
|  |  | ||||||
| 	bool contentSearch = query.hasContentSearch(); | 	bool contentSearch = query.hasContentSearch() || query.hasOutlineSearch(); | ||||||
| 	while(dbQuery.next()) | 	while(dbQuery.next()) | ||||||
| 	{ | 	{ | ||||||
| 		SearchResult result; | 		SearchResult result; | ||||||
|   | |||||||
| @@ -4,11 +4,11 @@ TagStripperProcessor::TagStripperProcessor() | |||||||
| { | { | ||||||
| } | } | ||||||
|  |  | ||||||
| QVector<PageData> TagStripperProcessor::process(const QByteArray &data) const | DocumentProcessResult TagStripperProcessor::process(const QByteArray &data) const | ||||||
| { | { | ||||||
| 	auto result = DefaultTextProcessor::process(data); | 	auto result = DefaultTextProcessor::process(data); | ||||||
| 	// TODO: does not work properly with <br> and does not deal with entities... | 	// TODO: does not work properly with <br> and does not deal with entities... | ||||||
|  | 	Q_ASSERT(result.pages.size() > 0); | ||||||
| 	result[0].content.remove(QRegExp("<[^>]*>")); | 	result.pages[0].content.remove(QRegExp("<[^>]*>")); | ||||||
| 	return result; | 	return result; | ||||||
| } | } | ||||||
|   | |||||||
| @@ -8,7 +8,7 @@ class TagStripperProcessor : public DefaultTextProcessor | |||||||
| 	TagStripperProcessor(); | 	TagStripperProcessor(); | ||||||
|  |  | ||||||
|   public: |   public: | ||||||
| 	QVector<PageData> process(const QByteArray &data) const override; | 	DocumentProcessResult process(const QByteArray &data) const override; | ||||||
| }; | }; | ||||||
|  |  | ||||||
| #endif // XMLSTRIPPERPROCESSOR_H | #endif // XMLSTRIPPERPROCESSOR_H | ||||||
|   | |||||||
| @@ -20,6 +20,7 @@ enum TokenType | |||||||
| 	FILTER_PATH_ENDS, | 	FILTER_PATH_ENDS, | ||||||
| 	FILTER_PATH_STARTS, | 	FILTER_PATH_STARTS, | ||||||
| 	FILTER_TAG_ASSIGNED, | 	FILTER_TAG_ASSIGNED, | ||||||
|  | 	FILTER_OUTLINE_CONTAINS, | ||||||
| 	FILTER_CONTENT = 512, /* Everything below here is content search (except LIMIT) */ | 	FILTER_CONTENT = 512, /* Everything below here is content search (except LIMIT) */ | ||||||
| 	FILTER_CONTENT_CONTAINS, | 	FILTER_CONTENT_CONTAINS, | ||||||
| 	FILTER_CONTENT_PAGE, | 	FILTER_CONTENT_PAGE, | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user