Initialize files with configurable threadpoolHEADmaster

This greatly speed-up the initialization of directories
with a lot of files inside.

2 files changed, 15 insertions, 2 deletions

diff --git a/fuse-httpfs b/fuse-httpfs
index 0963501..f68be8f 100755
--- a/fuse-httpfs
+++ b/fuse-httpfs
@@ -16,12 +16,14 @@ p.add_argument("--max_background", type=int, default=15,
                help="Maximum number of background threads")
 p.add_argument("--no_foreground", action="store_true", default=False,
                help="Fork into background as a daemon")
-p.add_argument("--debug", action="store_true", 
+p.add_argument("--debug", action="store_true",
                help="Enable debug logging")
 p.add_argument("--debug-requests", action="store_true",
                help="Enable python-requests debug logging")
 p.add_argument("--nothreads", action="store_true",
                help="Disable fuse threads")
+p.add_argument("--poolsize", type=int, default=httpfs.Config.poolsize,
+               help="Max no. of concurrent http requests")
 p.add_argument("--timeout", type=float, default=None,
                help="HTTP connect and read timeout")
 p.add_argument("--ssl", choices=["default", "system", "none"],
@@ -35,6 +37,7 @@ httpfs.Config.timeout = args.pop("timeout")
 httpfs.Config.mountpoint = args.pop("mountpoint")
 httpfs.Config.verify = args.pop("ssl")
 httpfs.Config.system_ca = args.pop("system_ca")
+httpfs.Config.poolsize = args.pop("poolsize")
 kwargs = {}
 if not args.pop("no_foreground"):
     kwargs["foreground"] = True
diff --git a/httpfs/__init__.py b/httpfs/__init__.py
index d877f3e..4e1e4ff 100644
--- a/httpfs/__init__.py
+++ b/httpfs/__init__.py
@@ -8,6 +8,7 @@ from email.utils import parsedate
 from html.parser import HTMLParser
 from stat import S_IFDIR, S_IFREG
 from errno import EIO, ENOENT, EBADF, EHOSTUNREACH
+from multiprocessing import dummy as mp
 
 import fuse
 import requests
@@ -15,11 +16,13 @@ import requests
 
 logger = logging.getLogger(__name__)
 
+
 class Config:
     mountpoint = None
     timeout = None
     verify = None
     system_ca = None
+    poolsize = 15
 
 
 class Path:
@@ -140,6 +143,11 @@ class Directory(Path):
         self.entries.update(parser.entries)
         r.close()
 
+        files = filter(lambda obj: isinstance(obj, File),
+                       self.entries.values())
+        pool = mp.Pool(Config.poolsize)
+        pool.map(lambda it: it.init(), files)
+
         logger.info("Diretory loaded {}".format(url))
         self.initialized = True
 
@@ -159,6 +167,8 @@ class Server(Directory):
         super().__init__(parent, name)
         self.session = requests.Session()
         self.session.allow_redirects = True
+        adapter = requests.adapters.HTTPAdapter(
+            pool_connections=Config.poolsize, pool_maxsize=Config.poolsize)
         if Config.verify == "default":
             pass
         elif Config.verify == "system":
@@ -297,7 +307,7 @@ class Httpfs(fuse.LoggingMixIn, fuse.Operations):
                                         allow_redirects=True)
                 if r.status_code == 200:
                     logger.info("Create directory for path: {} "
-                                 "at: {}".format(path, url))
+                                "at: {}".format(path, url))
                     prevEntry.entries[lastElement] = d
                 else:
                     logger.info("Path not found ({}): {} for {}".format(