fix content-length and better request bypassing

1 files changed, 52 insertions, 28 deletions

diff --git a/proxy.py b/proxy.py
index 99793a8..0d50d67 100755
--- a/proxy.py
+++ b/proxy.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python -t
-import  os, sys, string, time
+import  os, sys, string, time, md5
 import asynchat, asyncore, socket, httplib, urlparse
 from heapq import heappush, heappop
 try:
@@ -11,7 +11,7 @@ except ImportError:
 ENDPOINTS = [
     ('10.2.2.11', 8888),
     ('10.3.1.2', 8888),
-#    ('10.1.1.156', 8888),
+    ('10.1.1.156', 8888),
 ]
 
 kB = 1024
@@ -50,6 +50,9 @@ class Fetcher(asynchat.async_chat):
         self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
         self.connect(self.proxy)
 
+    def __str__(self):
+        return "<Fetcher proxy=%s url=%s range=%s" % (self.proxy, urlparse.urlunparse(self.url), self.range)
+
     def handle_connect (self):
         print self, "Start"
         self.send("GET http://%s:%s%s HTTP/1.0\r\n" % ( self.url.hostname, self.url.port or 80, self.url.path ))
@@ -93,9 +96,6 @@ class Fetcher(asynchat.async_chat):
             self.set_terminator(None)
             self.reader.handle_incoming_http_header(self.http_header)
 
-    def __str__(self):
-        return "<Fetcher proxy=%s url=%s range=%s" % (self.proxy, urlparse.urlunparse(self.url), self.range)
-
 class StringIOProducer(object):
     def __init__(self,buf, amt=1440):
         self.buf=buf
@@ -111,6 +111,7 @@ class MultipleProxyReader(object):
         self.url = url
         self.header = header
         self.content_length = content_length
+        print self, "New Instance"
         
         self.header_sent = False
         self.fetch_pos = 0
@@ -122,6 +123,9 @@ class MultipleProxyReader(object):
         for proxy in ENDPOINTS:
             self.fetchers.append( Fetcher(self, proxy, self.url, self.header, self.next_range(INIT_BLOCKSIZE)) )
 
+    def __str__(self):
+        return "<MultipleProxyReader url=%s content_length=%s>" % (urlparse.urlunparse(self.url), self.content_length)
+
     def handle_incoming_data(self, fetcher, data=None, length=0):
         if not data:
             self.fetchers = filter(lambda f: f != fetcher, self.fetchers)
@@ -172,15 +176,44 @@ class MultipleProxyReader(object):
             headers = httplib.HTTPMessage(header)
             for key in filter(lambda k: k not in ("content-range", "content-length"), headers.dict.keys()):
                 self.channel.push("%s: %s\r\n" % (key, headers.dict[key]))
-            self.channel.push("Content-Length: %s" % self.content_length)
+            self.channel.push("Content-Length: %s\r\n" % self.content_length)
             self.channel.push("X-Proxy: Magicproxy (superpower activated)\r\n")
             self.channel.push("\r\n")
 
+"""Transparent forward to other proxy server"""
+class HTTPProxyClient(asynchat.async_chat):
+    def __init__(self, proxy, channel, method, url, headers):
+        self.proxy = proxy
+        self.other = channel
+        self.method = method
+        self.headers = headers
 
+        asynchat.async_chat.__init__(self)
+        self.set_terminator(None)
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.connect(self.proxy)
 
+        self.buf = StringIO.StringIO()
+        self.buf.write("%s %s HTTP/1.0\r\n" % (method, urlparse.urlunparse(url)))
+        for key in headers.keys():
+            self.buf.write("%s: %s\r\n" % (key, headers[key]))
+        self.buf.write("\r\n")
+    
     def __str__(self):
-        return "<MultipleProxyReader url=%s content_length=%s>" % (urlparse.urlunparse(self.url), self.content_length)
+        return "<HTTPProxyClient proxy=%s:%s>" % self.proxy
 
+    def collect_incoming_data(self, data):
+        self.other.push(data)
+
+    def handle_close(self):
+        self.close()
+        self.other.close_when_done()
+#        print self, "Done"
+
+    def handle_connect(self):
+        self.push_with_producer(StringIOProducer(self.buf))
+
+        
 class HTTPChannel(asynchat.async_chat):
     def __init__(self, server, sock, addr):
         self.server = server
@@ -193,6 +226,7 @@ class HTTPChannel(asynchat.async_chat):
         self.set_terminator("\r\n\r\n")
 
     def handle_close(self):
+        self.close()
         self.is_closed = True
 
     def collect_incoming_data(self, data):
@@ -224,6 +258,9 @@ class HTTPProxyServer(asyncore.dispatcher):
         self.bind(("", 8080))
         self.listen(5)
 
+    def __str__(self):
+        return "<HTTPProxyServer port=%s>" % self.port
+
     def shutdown(self):
         #TODO Hier Proxy sauber beenden
         #self.channel.close_when_done()
@@ -246,32 +283,19 @@ class HTTPProxyServer(asyncore.dispatcher):
         content_length = filter(lambda it: it[0] == "content-length", resp.getheaders())
         if len( content_length ) == 0:
             # no content length given, bypass this request
-            print "missing content-length, bypass"
             self._bypass_request(channel, "GET", url, headers)
         else:
             content_length = int(content_length[0][1])
-
-        if content_length < THRESHOLD:
-            self._bypass_request(channel, "GET", url, headers)
-        else:
-            MultipleProxyReader(channel, url, headers, content_length)
+            if content_length < THRESHOLD:
+                self._bypass_request(channel, "GET", url, headers)
+            else:
+                MultipleProxyReader(channel, url, headers, content_length)
         
     def _bypass_request(self, channel, method, url, headers):
-        print "_bypass request: %s %s" % (method, urlparse.urlunparse(url))
-        #XXX hier sollte nicht proxy gespielt werden sondern
-        #die daten 1-zu-1 durchgereicht werden.
-        #Weiterhin sollte sichergestellt werden, dass die requests
-        #zu Host X1 immer ueber Proxy Y1 geroutet werden
-        # etwa proxy=proxies[ stuff(hostname) % len(proxies) ]
-        conn = httplib.HTTPConnection(url.hostname, url.port or 80)
-        conn.request(method, url.path, body="", headers=headers)
-        resp = conn.getresponse()
-        channel.push("HTTP/1.0 200 OK\r\nX-Proxy: Magicproxy (superpower disabled)\r\n")
-        channel.push( "\r\n".join(map(lambda k: "%s: %s" % (k[0],k[1]), resp.getheaders())) )
-        channel.push("\r\n\r\n")
-        channel.push(resp.read())
-        channel.close_when_done()
-
+        proxy = ENDPOINTS[ int( md5.md5(url.hostname).hexdigest(),16 ) % len(ENDPOINTS) ]
+        print self, "_bypass request via %s: %s %s" % (proxy, method, urlparse.urlunparse(url))
+        HTTPProxyClient(proxy, channel, method, url, headers)
+        
 if __name__ == "__main__":
     proxy = HTTPProxyServer()
     asyncore.loop()