parallel

1 files changed, 50 insertions, 51 deletions

diff --git a/imdb-lookup/imdbinfo.py b/imdb-lookup/imdbinfo.py
index 7457c98..b7300cc 100755
--- a/imdb-lookup/imdbinfo.py
+++ b/imdb-lookup/imdbinfo.py
@@ -13,7 +13,7 @@ import shutil
 import logging
 import hashlib
 from io import BytesIO
-
+from multiprocessing import Pool
 try:
     from PIL import Image, ImageFilter
     import tmdbsimple as tmdb
@@ -173,7 +173,7 @@ class TMDBCache(object):
             data = json.loads(r.text)
             r.close()
             assert data is not None and data['data'], "Data must not be empty"
-            return data
+            return data['data']
         return self._cache_json(movie_id, "imdb_maindetails", do_request)
 
     def omdb_movie(self, movie_id):
@@ -285,12 +285,7 @@ def do_rating(args, imdb_ids):
     for info in infos:
         print("{rating:.02f} {imdb_id} {title:30s} avg=" +
               "{vote_average:.1f} count={vote_count:.0f}".format(**info))
-
-
-def do_index(args, imdb_ids):
-    """creates a index website"""
-    valid_extensions = [".mkv", ".avi", ".mov", ".mp4", ".mpg"]
-
+def getInfo(a):
     def listMovieFiles(path):
         def listFiles():
             for root, dirs, files in os.walk(path):
@@ -302,6 +297,50 @@ def do_index(args, imdb_ids):
             files = [path]
         return files
 
+    def poster(imdb_id):
+        data = db.poster_low(imdb_id)
+        if data:
+            if not os.path.exists(".index.html/poster"):
+                os.makedirs(".index.html/poster")
+
+            out = os.path.join(".index.html/poster",imdb_id+".jpg")
+            open(out, "wb").write(data)
+            return out
+        else:
+            return None
+    path, imdb_id = a
+    db = TMDBCache() #Protector(TMDBCache())
+    imdb = db.imdb_movie(imdb_id)
+    omdb = db.omdb_movie(imdb_id)
+    tmdb = db.infos(imdb_id)
+    if not imdb or not omdb or not tmdb:
+        print("Error in {} {}".format(path, imdb_id))
+        return None
+    else:
+        return {
+        'id': imdb_id,
+        'title': tmdb['title'],
+        'path': path,
+        'poster': poster(imdb_id),
+        'tagline': 'tagline' in imdb and imdb['tagline'] or None,
+        'plot': 'plot' in imdb and imdb['plot']['outline'] or None,
+        'website': 'homepage' in imdb and imdb['homepage'] or omdb['Website'] != 'N/A' and omdb['Website'] or None,
+        'release': 'release_date' in tmdb and tmdb['release_date'] or None,
+        'movieFiles': list(listMovieFiles(path)),
+        'imdbRating': imdb['rating'],
+        'imdbVotes': imdb['num_votes'],
+        'omdbTomatoConsensus': (omdb['tomatoConsensus'] != 'N/A') and omdb['tomatoConsensus'] or None,
+        'omdbTomato': (omdb['tomatoMeter'] != 'N/A') and float(omdb['tomatoMeter']) or None,
+        'omdbUserTomato': (omdb['tomatoUserMeter'] != 'N/A') and float(omdb['tomatoUserMeter']) or None,
+        'omdbTomatoRating': (omdb['tomatoRating'] != 'N/A') and float(omdb['tomatoRating']) or None,
+        'omdbTomatoUserRating': (omdb['tomatoUserRating'] != 'N/A') and float(omdb['tomatoUserRating']) or None,
+        'tmdbId': tmdb['id'],
+    }
+
+def do_index(args, imdb_ids):
+    """creates a index website"""
+    valid_extensions = [".mkv", ".avi", ".mov", ".mp4", ".mpg"]
+
     def install(filename):
         outpath = os.path.join(".index.html", os.path.dirname(filename))
         if not os.path.exists(outpath):
@@ -312,48 +351,6 @@ def do_index(args, imdb_ids):
         return out
 
     def data(callbackName):
-        db = Protector(TMDBCache())
-        def poster(imdb_id):
-            data = db.poster_low(imdb_id)
-            if data:
-                if not os.path.exists(".index.html/poster"):
-                    os.makedirs(".index.html/poster")
-
-                out = os.path.join(".index.html/poster",imdb_id+".jpg")
-                open(out, "wb").write(data)
-                return out
-            else:
-                return None
-            
-        def getInfo(a):
-            path, imdb_id = a
-            imdb = db.imdb_movie(imdb_id)['data']
-            omdb = db.omdb_movie(imdb_id)
-            tmdb = db.infos(imdb_id)
-            if not imdb or not omdb or not tmdb:
-                print("Error in {} {}".format(path, imdb_id))
-                return None
-            else:
-                return {
-                'id': imdb_id,
-                'title': tmdb['title'],
-                'path': path,
-                'poster': poster(imdb_id),
-                'tagline': 'tagline' in imdb and imdb['tagline'] or None,
-                'plot': 'plot' in imdb and imdb['plot']['outline'] or None,
-                'website': 'homepage' in imdb and imdb['homepage'] or omdb['Website'] != 'N/A' and omdb['Website'] or None,
-                'release': 'release_date' in tmdb and tmdb['release_date'] or None,
-                'movieFiles': list(listMovieFiles(path)),
-                'imdbRating': imdb['rating'],
-                'imdbVotes': imdb['num_votes'],
-                'omdbTomatoConsensus': (omdb['tomatoConsensus'] != 'N/A') and omdb['tomatoConsensus'] or None,
-                'omdbTomato': (omdb['tomatoMeter'] != 'N/A') and float(omdb['tomatoMeter']) or None,
-                'omdbUserTomato': (omdb['tomatoUserMeter'] != 'N/A') and float(omdb['tomatoUserMeter']) or None,
-                'omdbTomatoRating': (omdb['tomatoRating'] != 'N/A') and float(omdb['tomatoRating']) or None,
-                'omdbTomatoUserRating': (omdb['tomatoUserRating'] != 'N/A') and float(omdb['tomatoUserRating']) or None,
-                'tmdbId': tmdb['id'],
-            }
-
         filename = ".index.html/js/data.js"
         if not os.path.exists(os.path.dirname(filename)):
             os.makedirs(os.path.dirname(filename))
@@ -361,7 +358,9 @@ def do_index(args, imdb_ids):
         out = open(filename, "w")
         out.write(callbackName)
         out.write("(")
-        json.dump(list(filter(bool, map(getInfo, imdb_ids))), out)
+        pool = Pool(processes=10) # parallel fetchers
+        infos = pool.map(getInfo, imdb_ids)
+        json.dump(list(filter(bool,infos)), out)
         out.write(")")
         out.close()
         return filename