#!/usr/bin/python2.6
# coding: utf-8
import os
import sys
import pyPdf
from whoosh.index import create_in, open_dir
import whoosh.fields as fields
import time
from cStringIO import StringIO
from Queue import Queue, Empty
import multiprocessing as mp

schema_book = fields.Schema(
    pagenumber=fields.NUMERIC(stored=True),
    metadata_docnum=fields.NUMERIC(stored=True),
    content=fields.TEXT(stored=True),
    )

schema_metadata = fields.Schema(
    title = fields.TEXT(stored=True),
    path=fields.ID(stored=True,unique=True),
    createtime=fields.NUMERIC(stored=True) )

if not os.path.exists(u"index"):
    create_index = True
    os.mkdir(u"index")
    index_book = create_in(u"index", schema_book, u"book")
    index_metadata = create_in(u"index", schema_metadata, u"metadata")
else:
    create_index = False
    index_book = open_dir(u"index", u"book")
    index_metadata = open_dir(u"index", u"metadata")


filepaths = []
directory = unicode(sys.argv[1], "utf8")
searcher_book = index_book.searcher()
searcher_metadata = index_metadata.searcher()
print u"Walking {0}".format(directory)
filecount = 0
skipped = 0
for path, directories, files in os.walk(directory):
    for filename in files:
        if filename.endswith(u".pdf"):
            filepath = os.path.join(path, filename)
            docnum = create_index or searcher_metadata.document_number(path=filepath)
            if not docnum:
                skipped += 1
            else:
                filepaths.append(filepath)
                filecount += 1
            print u"\r{0} files found {1} skipped".format(filecount+skipped, skipped),
print ""

if not create_index: #update index for deleted files
    writer_book = index_book.writer()
    writer_metadata = index_metadata.writer()
    deleted = 0
    processed = 0
    for fields in searcher_metadata.all_stored_fields():
        path = fields['path']
        processed += 1
        if not os.path.exists(path):
            writer_book.delete_by_term(u'metadata_docnum', searcher_metadata.document_number(path=path))
            writer_metadata.delete_by_term('path', path)
            deleted += 1
        print u"\r{0} pages processed. {1} deleted".format(processed, deleted),
    print ""
    writer_book.commit()
    writer_metadata.commit()

searcher_book.close()
searcher_metadata.close()

def process_file(filepath):
    print u"{0} processing {1}".format(os.getpid(), filepath)
    inputfile = pyPdf.PdfFileReader(file(filepath, 'r'))
    title = inputfile.getDocumentInfo().title
    writer_metadata = index_metadata.writer()
    writer_metadata.add_document(title=title, path=filepath, createtime=time.time())
    writer_metadata.commit()
    searcher_metadata = index_metadata.searcher()
    metadata_docnum = searcher_metadata.document_number(path=filepath)
    searcher_metadata.close()

    pagenumber = 1
    for page in inputfile.pages:
        print u"processing {0} Page {1}".format(filepath, pagenumber)
        content = page.extractText()
        writer_book = index_book.writer()
        writer_book.add_document(pagenumber=pagenumber,
                                  metadata_docnum=metadata_docnum,
                                  content=content)
        writer_book.commit()                         
        pagenumber += 1
try:
    pool = mp.Pool()
    pool.apply(process_file, filepaths)
except ImportError:
    for filepath in filepaths:
        process_file(filepath)
print u"optimize indexes"
index_metadata.optimize()
index_metadata.close()
index_book.optimze()
index_book.close()