Merge branch 'master' of git://github.com/yvesf/magicproxy

Conflicts:
	proxy.py

1 files changed, 41 insertions, 45 deletions

diff --git a/proxy.py b/proxy.py
index 963275f..a22a9cc 100644
--- a/proxy.py
+++ b/proxy.py
@@ -10,20 +10,20 @@ except ImportError:
 ENDPOINTS = [
     ('10.2.2.11', 8888),
     ('10.3.1.2', 8888),
+    ('10.1.1.156', 8888),
 ]
 
 #minimum entity size to start a paralel fetch
-MINIMAL_SIZE = 524288
-BLOCKSIZE_FETCH = 1048576
-
+KB = 1024
+MINIMAL_SIZE = 512 * KB
+BLOCKSIZE_FETCH = 2048 * KB
 class Fetcher(asynchat.async_chat):
-    def __init__(self, reader, proxy, url, header, range, content_length):
-        print "Fetcher using proxy=%s for url=%s range=%s content_length=%s" % (proxy, url, range, content_length)
+    def __init__(self, reader, proxy, url, header, range):
         self.reader = reader
         self.proxy = proxy
         self.url = url
+        self.header = self.header
         self.range = range
-        self.content_length = content_length
 
         self.start_time = time.time()
 
@@ -38,28 +38,23 @@ class Fetcher(asynchat.async_chat):
 
     #XXXshould include original request headers
     def handle_connect (self):
-        print "handle_connect"
-        self.debug_send("GET http://%s:%s%s HTTP/1.0\r\n" % ( self.url.hostname, self.url.port or 80, self.url.path ))
-        self.debug_send("Range: bytes=%s-%s\r\n" % (self.range[0], self.range[1]))
-        self.debug_send("\r\n")
-        print "sent request"
-        
-    def debug_send(self, data):
-        print "SEND", data
-        self.send(data)
+        print self,"handle_connect"
+        self.send("GET http://%s:%s%s HTTP/1.0\r\n" % ( self.url.hostname, self.url.port or 80, self.url.path ))
+        #XXX self.header
+        self.send("Range: bytes=%s-%s\r\n" % (self.range[0], self.range[1]))
+        self.send("\r\n")
     
     def time(self):
         return self.stop_time - self.start_time
 
     def collect_incoming_data(self, data):
-        if self.state==2:#body
-            print "read at %s - range_start=%s range_end=%s" % (self.pos, self.range[0], self.range[1])
+        if self.state==2: #body
             self.reader.handle_incoming_data(self.pos, data)
             self.pos += len(data)
             if self.pos >= self.range[1]:
                 self.reader.finished()
-                print "fetcher finished"
                 self.stop_time = time.time()
+                print self, "finished"
                 self.close_when_done()
         elif self.state ==1: #header
             self.http_header += data
@@ -70,13 +65,16 @@ class Fetcher(asynchat.async_chat):
         if self.state == 0:
             self.state = 1
             self.set_terminator("\r\n\r\n")
-            print "got status line"
+            #print "got status line"
         elif self.state == 1:
             self.state = 2
             self.set_terminator(None)
-            print "header ends now"
+            print self, "header ends now"
             self.reader.handle_incoming_http_header(self.http_header)
 
+    def __str__(self):
+        return "<Fetcher proxy=%s url=%s range=%s" % (self.proxy, self.url, self.range)
+
 class MultipleProxyReader(object):
     def __init__(self, channel, url, header, content_length):
         print "MultipleProxyReader url=%s content_length=%s" % (url, content_length)
@@ -96,7 +94,6 @@ class MultipleProxyReader(object):
             self.finished() #bad-named XXX
 
     def handle_incoming_data(self, pos, data):
-        print "got body data at pos %s" % pos
         self.blocks[pos] = data
 
         while self.find_next_data():
@@ -108,24 +105,19 @@ class MultipleProxyReader(object):
         else:
             self.header_sent = True
 
-            self.debug_send("HTTP/1.0 200 OK\r\n")
+            self.channel.push("HTTP/1.0 200 OK\r\n")
+            # Sends header from first response
             headers = httplib.HTTPMessage(StringIO.StringIO(header))
-            for key in headers.dict.keys():
-                if key in ("content-range", "content-length"):
-                    continue
-                self.debug_send("%s: %s\r\n" % (key, headers.dict[key]))
-            self.debug_send("Content-Length: %s" % self.content_length)
-            self.debug_send("X-Proxy: Magicproxy (superpower activated)\r\n")
-            self.debug_send("\r\n")
-
-    def debug_send(self, data):
-        print "SEND", data
-        self.channel.push(data)
+            for key in filter(lambda k: k not in ("content-range", "content-length"), headers.dict.keys()):
+                self.channel.push("%s: %s\r\n" % (key, headers.dict[key]))
+            self.channel.push("Content-Length: %s" % self.content_length)
+            self.channel.push("X-Proxy: Magicproxy (superpower activated)\r\n")
+            self.channel.push("\r\n")
 
     def finished(self):
         self.find_next_data()
         if self.fetch_pos +1 <self.content_length:
-            print "start new fetcher"
+            print self, "start new fetcher"
             start = self.fetch_pos
             if self.fetch_pos+BLOCKSIZE_FETCH < self.content_length:
                 self.fetch_pos += BLOCKSIZE_FETCH
@@ -136,20 +128,28 @@ class MultipleProxyReader(object):
             self.next_endpoint = (self.next_endpoint+1) % len(ENDPOINTS)
 
     def find_next_data(self):
+
+        if not self.channel.writeable(): # request-side closed the connection
+            self.channel.close_when_done()
+            return False
+
         r = False
-        print "expect at %s" % self.write_pos, self.blocks.keys()
+        #print "expect at %s" % self.write_pos, self.blocks.keys()
         if self.write_pos in self.blocks.keys():
             r=True
             data = self.blocks.pop(self.write_pos)
             self.channel.push(data)
             self.write_pos += len(data)
-            print "Send %s bytes" % len(data)
+            #print "Send %s bytes" % len(data)
 
         if self.write_pos + 1 >= self.content_length:
-            print "done"
+            print self, "done"
             self.channel.close_when_done()
         return r
 
+    def __str__(self):
+        return "<MultipleProxyReader >"
+
 class HTTPChannel(asynchat.async_chat):
     def __init__(self, server, sock, addr):
         asynchat.async_chat.__init__(self, sock)
@@ -205,16 +205,12 @@ class HTTPProxyServer(asyncore.dispatcher):
         if len( content_length ) == 0:
             # no content length given, bypass this request
             print "missing content-length, bypass"
-            return self._bypass_request(channel, "GET", url)
+            self._bypass_request(channel, "GET", url)
         else:
             content_length = int(content_length[0][1])
-
-        #if content_length < MINIMAL_SIZE:
-        #    return self._bypass_request(channel, "GET", url)
-
-        ##XXX
-        header = (("foo", "bla"))
-        MultipleProxyReader(channel, url, header, content_length)
+            ##XXX
+            header = (("foo", "bla"))
+            MultipleProxyReader(channel, url, header, content_length)
         
     def _bypass_request(self, channel, method, url):
         print "_bypass request hostname=%s port=%s path=%s" % (url.hostname, url.port or 80, url.path)