1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
|
#!/usr/bin/python2.6
# coding: utf-8
import os
import sys
import pyPdf
from whoosh.index import create_in, open_dir
import whoosh.fields as fields
import time
from cStringIO import StringIO
from Queue import Queue, Empty
from threading import Thread, Condition
schema = fields.Schema(
title=fields.TEXT(stored=True),
path=fields.ID(stored=True),
content=fields.TEXT(stored=True),
createtime=fields.NUMERIC() )
if not os.path.exists("index"):
os.mkdir("index")
index = create_in(u"index", schema)
else:
index = open_dir("index")
filepaths = Queue()
documents = Queue()
notifier = Condition()
directory = unicode(sys.argv[1], "utf8")
print u"Walking {0}".format(directory)
filecount = 0
for path, directories, files in os.walk(directory):
for filename in files:
if filename.endswith(".pdf"):
filepaths.put(os.path.join(path, filename))
filecount += 1
print u"\r{0} files found".format(filecount),
print ""
class PDFWorker(Thread):
def run(self):
while True:
try:
filepath = filepaths.get(False)
except Empty:
break
try:
print u"{0} processing {1}".format(self.name, filepath)
inputfile = pyPdf.PdfFileReader(file(filepath, 'r'))
title = inputfile.getDocumentInfo().title
content = u""
for page in inputfile.pages:
content += page.extractText()
documents.put( {"title":title, "path":filepath, "content":content, "createtime":time.time() } )
except Exception, e:
print u"{0} Exception: {1}".format(self.name, str(e))
finally:
print u"{0} finished {1}".format(self.name, filepath)
filepaths.task_done()
class IndexWorker(Thread):
def run(self):
while index != None:
try:
doc = documents.get(True, 0.5)
except Empty:
continue
print u"{0} adding {1}".format(self.name, doc['path'])
writer = index.writer()
writer.add_document(**doc)
writer.commit()
documents.task_done()
print u"{0} added {1}".format(self.name, doc['path'])
threads = map(lambda i: PDFWorker(), range(1))
for thread in threads:
thread.start()
idx = IndexWorker()
idx.start()
print "all running"
for thread in threads:
thread.join()
oldindex = index
index = None
print "optimize index"
oldindex.optimize()
oldindex.close()
"""
try:
filepath = os.path.join(path, filename)
print u"Process {0}".format(filepath)
inputfile = pyPdf.PdfFileReader(file(filepath, 'r'))
title = inputfile.getDocumentInfo().title
i=1
content = ""
numpages = inputfile.getNumPages()
for page in inputfile.pages:
sys.stdout.write("\rPage {0}/{1}".format(i, numpages))
sys.stdout.flush()
content += page.extractText()
i+=1
print u""
writer = index.writer()
writer.add_document(title=title, path=filepath, content=content, createtime=time.time())
writer.commit()
except Exception,e:
print e
"""
|
