added multiprocess support (quick hack)

addindex

@@ -8,7 +8,8 @@ import xml.etree.ElementTree
 import re
 import chardet
 import config
-dbcon = sqlite3.connect(config.DBPATH, isolation_level=None)
+from multiprocessing import Pool
+
 
 
 class pagedata:
@@ -131,6 +132,8 @@ def insert(path, cursor):
 		processor=preprocess[ext]
 	pagedatalist = processor(abspath)
 
+	#TODO: assumes sqlitehas been built with thread safety (and it is the default)
+	cursor = dbcon.cursor()
 	cursor.execute("BEGIN TRANSACTION")
 	cursor.execute("DELETE FROM file WHERE path = ?", (abspath,))
 	cursor.execute("INSERT INTO file(path, mtime) VALUES(?, ?) ", (abspath, mtime))
@@ -143,14 +146,31 @@ preprocess={".pdf":process_pdf, ".odt":process_odt, ".html":process_striptags, "
 			".sql":process_text, ".c":process_text, ".cpp":process_text, ".js":process_text, ".java":process_text, 
 			".py":process_text, '.md':process_text}	
 
-cursor = dbcon.cursor()
-if len(sys.argv) < 2:
+
+
+def yieldstdinfiles():
 	for line in sys.stdin:
-		insert(line.replace("\n", ""), cursor)
-else:
-	for inputfile in sys.argv[1:]:
-		insert(inputfile, cursor)
-dbcon.close()
+		yield line.replace("\n", "")
+def poolinserter(path):
+	insert(path, cursor)
+	
+def init():
+	global cursor
+	global dbcon
+	dbcon = sqlite3.connect(config.DBPATH, isolation_level=None)
+
+
+cursor = None
+dbcon = None
+if __name__ == '__main__':
+	with Pool(processes=4,initializer=init) as pool:
+		
+		if len(sys.argv) < 2:
+			pool.map(poolinserter, yieldstdinfiles)
+		else:
+			pool.map(poolinserter, sys.argv[1:])
+
+