cli: CommandAdd: Implement --no-content and --fill-content

This commit is contained in:
Albert S. 2023-01-08 17:37:28 +01:00
부모 71789b5b56
커밋 8298b675aa
7개의 변경된 파일62개의 추가작업 그리고 17개의 파일을 삭제

파일 보기

@ -44,20 +44,30 @@ int CommandAdd::handle(QStringList arguments)
"Continue adding files, don't exit on first error. If this option is not given, looqs will "
"exit asap, but it's possible that a few files will still be processed. "
"Set -t 1 to avoid this behavior, but processing will be slower. "},
{{"n", "no-content"}, "Only add paths to database. Do not index content"},
{{"f", "fill-content"}, "Index content for files previously indexed with -n"},
{"tags", "Comma-separated list of tags to assign"},
{{"t", "threads"}, "Number of threads to use.", "threads"}});
parser.addHelpOption();
parser.addPositionalArgument("add", "Add paths to the index",
"add [paths...]. If no path is given, read from stdin, one path per line.");
parser.process(arguments);
this->keepGoing = parser.isSet("continue");
bool pathsOnly = parser.isSet("no-content");
bool fillContent = parser.isSet("fill-content");
if(parser.isSet("threads"))
{
QString threadsCount = parser.value("threads");
QThreadPool::globalInstance()->setMaxThreadCount(threadsCount.toInt());
}
if(pathsOnly && fillContent)
{
Logger::error() << "Invalid options: -n and -f cannot both be set";
return EXIT_FAILURE;
}
QStringList files = parser.positionalArguments();
if(files.length() == 0)

파일 보기

@ -119,11 +119,29 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
{
if(canonicalPath.startsWith(excludedPath))
{
if(this->fileSaverOptions.verbose)
{
Logger::info() << "Skipped due to excluded path";
}
return SKIPPED;
}
}
if(fileInfo.size() > 0)
bool mustFillContent = this->fileSaverOptions.fillPathsOnlyWithContent;
if(!mustFillContent)
{
mustFillContent = !this->fileSaverOptions.pathsOnly;
if(mustFillContent)
{
auto filetype = this->dbService->queryFileType(fileInfo.absolutePath());
if(filetype)
{
mustFillContent = filetype.value() == 'c';
}
}
}
if(fileInfo.size() > 0 && mustFillContent)
{
QProcess process;
QStringList args;
@ -158,7 +176,7 @@ SaveFileResult FileSaver::saveFile(const QFileInfo &fileInfo)
}
}
}
SaveFileResult result = this->dbService->saveFile(fileInfo, pageData);
SaveFileResult result = this->dbService->saveFile(fileInfo, pageData, this->fileSaverOptions.pathsOnly);
if(result == OK && processorReturnCode == OK_WASEMPTY)
{
return OK_WASEMPTY;

파일 보기

@ -12,6 +12,7 @@ FileScanWorker::FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &qu
void FileScanWorker::run()
{
FileSaver saver{*this->dbService};
saver.setFileSaverOptions(this->fileSaverOptions);
auto paths = queue->dequeue(batchsize);
for(QString &path : paths)
{
@ -34,3 +35,8 @@ void FileScanWorker::run()
}
emit finished();
}
void FileScanWorker::setFileSaverOptions(FileSaverOptions options)
{
this->fileSaverOptions = options;
}

파일 보기

@ -15,12 +15,14 @@ class FileScanWorker : public QObject, public QRunnable
protected:
SqliteDbService *dbService;
ConcurrentQueue<QString> *queue;
FileSaverOptions fileSaverOptions;
int batchsize;
std::atomic<bool> *stopToken;
public:
FileScanWorker(SqliteDbService &db, ConcurrentQueue<QString> &queue, int batchsize, std::atomic<bool> &stopToken);
void run() override;
void setFileSaverOptions(FileSaverOptions options);
signals:
void result(FileScanResult);
void finished();

파일 보기

@ -98,6 +98,7 @@ void Indexer::launchWorker(ConcurrentQueue<QString> &queue, int batchsize)
FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize, this->workerCancellationToken);
connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult);
connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker);
runnable->setFileSaverOptions(this->fileSaverOptions);
++this->runningWorkers;
QThreadPool::globalInstance()->start(runnable);
}

파일 보기

@ -164,11 +164,15 @@ bool SqliteDbService::insertToFTS(bool useTrigrams, QSqlDatabase &db, int fileid
return true;
}
SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData)
SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly)
{
QString absPath = fileInfo.absoluteFilePath();
auto mtime = fileInfo.lastModified().toSecsSinceEpoch();
QChar fileType = fileInfo.isDir() ? 'd' : 'f';
QChar fileType = fileInfo.isDir() ? 'd' : 'c';
if(pathsOnly)
{
fileType = 'f';
}
QSqlDatabase db = dbFactory->forCurrentThread();
QSqlQuery delQuery(db);
@ -202,19 +206,23 @@ SaveFileResult SqliteDbService::saveFile(QFileInfo fileInfo, QVector<PageData> &
return DBFAIL;
}
int lastid = inserterQuery.lastInsertId().toInt();
if(!insertToFTS(false, db, lastid, pageData))
if(!pathsOnly)
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
}
if(!insertToFTS(true, db, lastid, pageData))
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
int lastid = inserterQuery.lastInsertId().toInt();
if(!insertToFTS(false, db, lastid, pageData))
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
}
if(!insertToFTS(true, db, lastid, pageData))
{
db.rollback();
Logger::error() << "Failed to insert data to FTS index " << Qt::endl;
return DBFAIL;
}
}
if(!db.commit())
{
db.rollback();

파일 보기

@ -19,7 +19,7 @@ class SqliteDbService
public:
SqliteDbService(DatabaseFactory &dbFactory);
SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData);
SaveFileResult saveFile(QFileInfo fileInfo, QVector<PageData> &pageData, bool pathsOnly);
unsigned int getFiles(QVector<FileData> &results, QString wildCardPattern, int offset, int limit);
bool deleteFile(QString path);
bool fileExistsInDatabase(QString path);