diff --git a/addindex b/addindex index e0ae2eb..0cf9d62 100755 --- a/addindex +++ b/addindex @@ -8,7 +8,8 @@ import xml.etree.ElementTree import re import chardet import config -dbcon = sqlite3.connect(config.DBPATH, isolation_level=None) +from multiprocessing import Pool + class pagedata: @@ -131,6 +132,8 @@ def insert(path, cursor): processor=preprocess[ext] pagedatalist = processor(abspath) + #TODO: assumes sqlitehas been built with thread safety (and it is the default) + cursor = dbcon.cursor() cursor.execute("BEGIN TRANSACTION") cursor.execute("DELETE FROM file WHERE path = ?", (abspath,)) cursor.execute("INSERT INTO file(path, mtime) VALUES(?, ?) ", (abspath, mtime)) @@ -143,14 +146,31 @@ preprocess={".pdf":process_pdf, ".odt":process_odt, ".html":process_striptags, " ".sql":process_text, ".c":process_text, ".cpp":process_text, ".js":process_text, ".java":process_text, ".py":process_text, '.md':process_text} -cursor = dbcon.cursor() -if len(sys.argv) < 2: + + +def yieldstdinfiles(): for line in sys.stdin: - insert(line.replace("\n", ""), cursor) -else: - for inputfile in sys.argv[1:]: - insert(inputfile, cursor) -dbcon.close() + yield line.replace("\n", "") +def poolinserter(path): + insert(path, cursor) + +def init(): + global cursor + global dbcon + dbcon = sqlite3.connect(config.DBPATH, isolation_level=None) + + +cursor = None +dbcon = None +if __name__ == '__main__': + with Pool(processes=4,initializer=init) as pool: + + if len(sys.argv) < 2: + pool.map(poolinserter, yieldstdinfiles) + else: + pool.map(poolinserter, sys.argv[1:]) + +