1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
|
#!/usr/bin/python2.6
# coding: utf-8
from __future__ import with_statement
import os
import time
try:
import json
except ImportError:
import simplejson as json
from StringIO import StringIO
from whoosh.index import open_dir
from whoosh.qparser import QueryParser
import whoosh.searching as searching
import whoosh.analysis as analysis
import whoosh.highlight as highlight
import whoosh.query as query
import flask
import pyPdf
import werkzeug
from werkzeug.contrib.cache import SimpleCache
import subprocess
from compat import str_format
app = flask.Flask("booksearch")
app.debug = True
image_cache = SimpleCache()
index_book = open_dir(u"index", u"book")
index_metadata = open_dir(u"index", u"metadata")
@app.route("/")
def do_index():
return flask.redirect(flask.url_for("do_search",term=""))
@app.route("/book/file/<int:docnum>")
def do_book_file(docnum):
with index_metadata.reader() as reader:
document = reader.stored_fields(docnum)
r = werkzeug.Response(open(document["path"], "r"), mimetype="application/pdf",)
r.headers.add("Content-Disposition", "attachment",
filename=os.path.basename(document["path"]))
return r
def pdf_to_image(filepath, page, size):
if not type(size) == int or size < 1 or size > 2000:
size = 260
density = 60 + 0.15 * size
app.logger.debug( str_format("Convert PDF to image page={page} size={size} density={density} filepath={filepath}", page=page, size=size, density=density, filepath=filepath ))
if image_cache.get("PDF_IMAGE_"+str(size)+"_"+str(page)+filepath) is None:
try:
pdffile = StringIO()
page = pyPdf.PdfFileReader(file(filepath, "r")).getPage(page)
out = pyPdf.PdfFileWriter()
out.addPage(page)
out.write(pdffile)
pdffile.seek(0)
process = subprocess.Popen(["/usr/local/bin/convert", "-density", str(density), "-resize", str(size), "pdf:-", "jpeg:-"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
process.stdin.write(pdffile.read())
image_data = ""
while not process.stdout.closed:
image_data += process.communicate()[0]
image_cache.set("PDF_IMAGE_"+str(size)+"_"+str(page)+filepath, image_data)
except Exception,e:
process = subprocess.Popen(["/usr/local/bin/convert", "-size", str(size), "-fill", "black", "-pointsize", "23", "caption:@-", "jpeg:-"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
message = str_format("Error:\n{error}", error=str(e))
if image_cache.get("EXCEPTION_"+ str(size) + message) is None:
image_data =""
process.stdin.write(message)
while not process.stdout.closed:
image_data += process.communicate()[0]
image_cache.set("EXCEPTION_" + str(size) + message, image_data, timeout=60*60)
return werkzeug.Response(image_cache.get("EXCEPTION_" + str(size) + message), mimetype="image/jpeg")
return werkzeug.Response(image_cache.get("PDF_IMAGE_"+str(size)+"_"+str(page)+filepath), mimetype="image/jpeg")
@app.route("/page/image/<int:docnum>", methods=["GET"])
@app.route("/page/image/<int:docnum>/<int:size>", methods=["GET"])
def do_page_image(docnum,size=260):
with index_book.reader() as reader:
document = reader.stored_fields(docnum)
page = document["pagenumber"] - 1
return pdf_to_image(document["path"], page, size=size)
@app.route("/book/frontpage/<int:docnum>", methods=["GET"])
def do_book_frontpage(docnum):
with index_metadata.reader() as reader:
document = reader.stored_fields(docnum)
return pdf_to_image(document["path"], 0, 260)
@app.route("/page/file/<int:docnum>", methods=["GET"])
def do_page_file(docnum):
with index_book.reader() as reader:
document = reader.stored_fields(docnum)
filepath = document["path"]
page = document["pagenumber"] - 1
app.logger.debug(str_format("Extract page={page} from filepath={filepath}", page=page, filepath=filepath))
inputfile = pyPdf.PdfFileReader(file(filepath, "r"))
pdfpage = inputfile.getPage(page)
outbuf = StringIO()
outfile = pyPdf.PdfFileWriter()
outfile.addPage(pdfpage)
outfile.write(outbuf)
outbuf.seek(0)
r = werkzeug.Response(outbuf, mimetype="application/pdf")
client_filename = os.path.basename(filepath)[:-3]
client_filename += str_format(u".Page-{page}.pdf", page=page)
r.headers.add("Content-Disposition", "attachment", filename=client_filename)
return r
class MyHtmlFormatter(highlight.HtmlFormatter):
def _format_fragment(self, text, fragment, seen):
text = unicode( flask.Markup.escape(text) )
return highlight.HtmlFormatter._format_fragment(self, text, fragment, seen)
@app.route("/excerpt/<int:docnum>/<path:term>", methods=["GET"])
def do_excerpt(docnum, term):
term = term.lower()
with index_book.reader() as reader:
document = reader.stored_fields(docnum)
q = QueryParser("content").parse(term)
q = q.simplify(reader)
terms = [ text for fieldname, text in q.all_terms()
if fieldname == "content" ]
excerpt = highlight.highlight(document["content"],
terms,
analysis.StandardAnalyzer(),
highlight.ContextFragmenter(terms, surround=40),
MyHtmlFormatter())
return unicode( excerpt )
@app.route("/search/<path:term>",methods=["GET"])
@app.route("/search/", methods=["GET"])
def do_search(term=None):
if not term:
return flask.render_template("search.html", objects=[], term="")
time_start = time.time()
term = term.lower()
searcher = index_book.searcher()
query = QueryParser("content").parse(term)
facets = searching.Facets.from_field(searcher, "path")
results = searcher.search(query, limit=None, sortedby="pagenumber")
categories = facets.categorize(results)
searcher.close()
books = {}
with index_book.reader() as reader:
for book in categories.items():
filepath = book[0]
with index_metadata.searcher() as searcher:
docnum = searcher.document_number(path=filepath)
with index_metadata.reader() as metadata_reader:
title = metadata_reader.stored_fields(docnum).get("title")
books[docnum] = {
"matches" : [],
"title":title,
"filename" : os.path.basename(filepath),
}
for match in book[1]:
pagenumber = reader.stored_fields(match[0])["pagenumber"]
match = (match[0], match[1], pagenumber)
books[docnum]["matches"].append(match)
return flask.render_template("search.html",
books=books,
term=term,
time_taken=time.time()-time_start)
def log_response(sender, response):
sender.logger.debug("Request context is about to close down. "
"Response: %s", response)
if __name__ == "__main__":
app.run(host="0.0.0.0", port=8000)
|
