diff --git a/shared/indexer.cpp b/shared/indexer.cpp new file mode 100644 index 0000000..3dc49bc --- /dev/null +++ b/shared/indexer.cpp @@ -0,0 +1,117 @@ +#include "indexer.h" +#include "logger.h" + +Indexer::Indexer(SqliteDbService &db) +{ + dirScanner = QSharedPointer(new ParallelDirScanner()); + connect(dirScanner.data(), &ParallelDirScanner::scanComplete, this, &Indexer::dirScanFinished); + connect(dirScanner.data(), &ParallelDirScanner::progress, this, &Indexer::dirScanProgress); + this->db = &db; +} + +void Indexer::beginIndexing() +{ + this->runningWorkers = 0; + this->currentScanProcessedCount = 0; + this->currentIndexResult = IndexResult(); + this->currentIndexResult.begin = QDateTime::currentDateTime(); + QVector dirs; + + for(QString &path : this->pathsToScan) + { + QFileInfo info{path}; + if(info.isDir()) + { + dirs.append(path); + } + else + { + this->filePathTargetsQueue.enqueue(path); + } + } + this->dirScanner->setPaths(dirs); + this->dirScanner->setIgnorePatterns(this->ignorePattern); + + this->dirScanner->scan(); + + launchWorker(this->filePathTargetsQueue, this->filePathTargetsQueue.remaining()); +} + +void Indexer::setIgnorePattern(QStringList ignorePattern) +{ + this->ignorePattern = ignorePattern; +} + +void Indexer::setTargetPaths(QVector pathsToScan) +{ + this->pathsToScan = pathsToScan; +} + +IndexResult Indexer::getResult() +{ + return this->currentIndexResult; +} + +void Indexer::dirScanFinished() +{ + Logger::info() << "Dir scan finished"; +} + +void Indexer::launchWorker(ConcurrentQueue &queue, int batchsize) +{ + FileScanWorker *runnable = new FileScanWorker(*this->db, queue, batchsize); + connect(runnable, &FileScanWorker::result, this, &Indexer::processFileScanResult); + connect(runnable, &FileScanWorker::finished, this, &Indexer::processFinishedWorker); + ++this->runningWorkers; + QThreadPool::globalInstance()->start(runnable); +} + +void Indexer::dirScanProgress(int current, int total) +{ + launchWorker(this->dirScanner->getResults(), current); + emit pathsCountChanged(total); +} + +void Indexer::processFileScanResult(FileScanResult result) +{ + if(verbose) + { + this->currentIndexResult.results.append(result); + } + else + { + if(result.second == DBFAIL || result.second == PROCESSFAIL || result.second == NOTFOUND) + { + this->currentIndexResult.results.append(result); + } + } + if(result.second == OK) + { + ++this->currentIndexResult.addedPaths; + } + else if(result.second == SKIPPED) + { + ++this->currentIndexResult.skippedPaths; + } + else + { + ++this->currentIndexResult.erroredPaths; + } + + if(currentScanProcessedCount++ == progressReportThreshold) + { + emit indexProgress(this->currentIndexResult.total(), this->currentIndexResult.addedPaths, + this->currentIndexResult.skippedPaths, this->currentIndexResult.erroredPaths, + this->dirScanner->pathCount()); + currentScanProcessedCount = 0; + } +} + +void Indexer::processFinishedWorker() +{ + --this->runningWorkers; + if(this->runningWorkers == 0 && !this->dirScanner->isRunning()) + { + emit finished(); + } +} diff --git a/shared/indexer.h b/shared/indexer.h new file mode 100644 index 0000000..374543e --- /dev/null +++ b/shared/indexer.h @@ -0,0 +1,86 @@ +#ifndef INDEXER_H +#define INDEXER_H +#include +#include +#include "sqlitedbservice.h" +#include "paralleldirscanner.h" +#include "filescanworker.h" + +class IndexResult +{ + public: + QDateTime begin; + QDateTime end; + QVector results; + + unsigned int addedPaths = 0; + unsigned int skippedPaths = 0; + unsigned int erroredPaths = 0; + + unsigned int total() + { + return addedPaths + skippedPaths + erroredPaths; + } + + QVector failedPaths() const + { + QVector result; + std::for_each(results.begin(), results.end(), + [&result](FileScanResult res) + { + if(res.second == DBFAIL || res.second == PROCESSFAIL || res.second == NOTFOUND) + { + result.append(res.first); + } + }); + return result; + } +}; + +class Indexer : public QObject +{ + Q_OBJECT + protected: + bool verbose = false; + bool keepGoing = true; + SqliteDbService *db; + + int progressReportThreshold = 50; + int currentScanProcessedCount = 0; + int runningWorkers = 0; + + QVector pathsToScan; + QSharedPointer dirScanner; + QSharedPointer fileScanner; + + QStringList ignorePattern; + + /* Those path pointing to files not directories */ + ConcurrentQueue filePathTargetsQueue; + + IndexResult currentIndexResult; + void launchWorker(ConcurrentQueue &queue, int batchsize); + + public: + void beginIndexing(); + void setIgnorePattern(QStringList ignorePattern); + void setTargetPaths(QVector pathsToScan); + + Indexer(SqliteDbService &db); + IndexResult getResult(); + + public slots: + void dirScanFinished(); + void dirScanProgress(int current, int total); + void processFileScanResult(FileScanResult result); + void processFinishedWorker(); + + signals: + void pathsCountChanged(int total); + void fileScanResult(FileScanResult *result); + void indexProgress(unsigned int processedFiles, unsigned int added, unsigned int skipped, unsigned int failed, + unsigned int totalPaths); + void finished(); +}; + +#endif // INDEXER_H