diff options
| -rwxr-xr-x | magicproxy/__init__.py | 45 |
1 files changed, 30 insertions, 15 deletions
diff --git a/magicproxy/__init__.py b/magicproxy/__init__.py index 5ab9773..8a78a54 100755 --- a/magicproxy/__init__.py +++ b/magicproxy/__init__.py @@ -5,13 +5,15 @@ from heapq import heappush, heappop import cStringIO as StringIO kB = 1024 - class DefaultConfiguration: """bind to that""" listen=("",8080) """available http-proxies""" - endpoints=[ ('10.2.2.11', 8888), ('10.3.1.2',8888) ] + endpoints=[ + ('10.2.2.11', 8888), + #('10.3.1.2',8888) + ] """minimum entity size to start parallelize fetch""" threshold = 512*kB @@ -32,12 +34,12 @@ class DefaultConfiguration: ################# class Fetcher(asynchat.async_chat): - def __init__(self, reader, proxy, url, headers, range): + def __init__(self, reader, proxy, url, headers, fetch_range): self.reader = reader self.proxy = proxy self.url = url self.headers = headers - self.range = range + self.range = fetch_range self.pos = (self.range[0] != -1) and self.range[0] or 0 self.start_time = 0 @@ -103,12 +105,20 @@ class Fetcher(asynchat.async_chat): def found_terminator(self): if self.state == 0: #got status-line + status = self.http_status.split(" ") + if len(status) > 1: + try: + self.http_status_code = int(status[1]) + except: + self.http_status_code = 520 #Bad Gateway + else: + self.http_status_code = 520 #Bad Gateway self.state = 1 - self.set_terminator("\r\n\r\n") + self.set_terminator("\r\n\r\n") #end of header elif self.state == 1: #got headers self.state = 2 self.set_terminator(None) - self.reader.handle_incoming_http_header(self, self.http_header) + self.reader.handle_incoming_http_header(self, self.http_status_code, self.http_header) class MagicHTTPProxyClient(object): def __init__(self, channel, url, header): @@ -176,11 +186,13 @@ class MagicHTTPProxyClient(object): self.fetch_pos = min(self.fetch_pos + suggested_blocksize, self.content_length) return (start, self.fetch_pos-1) - def handle_incoming_http_header(self, fetcher, header): + def handle_incoming_http_header(self, fetcher, status_code, header): if not self.channel.connected: return if self.header_sent: - pass + if status_code < 200 or status_code >= 300: + print self, "Error: got error code %s in %s. Giving up" % (status_code, fetcher) + self.channel.close() else: self.header_sent = True @@ -189,6 +201,8 @@ class MagicHTTPProxyClient(object): headers = httplib.HTTPMessage(header) content_length = filter(lambda i: i == "content-length", headers.dict.keys()) + #if there are content-length headers decide if entity size is + #bigger then threshold, if true then start n proxies (n=#endpoints) if len(content_length) == 1: content_length = int(headers.dict["content-length"]) if content_length >= self.config.threshold: @@ -202,13 +216,14 @@ class MagicHTTPProxyClient(object): else: content_length = None - buf = "HTTP/1.1 200 OK\r\n" - for key in filter(lambda k: k not in ("content-range", "content-length"), headers.dict.keys()): - buf += "%s: %s\r\n" % (key, headers.dict[key]) + buf = "HTTP/1.1 %s OK\r\n" % (status_code) + buf += "".join(map(lambda key: "%s: %s\r\n" % (key, headers.dict[key]), + filter(lambda k: k not in ("content-range", "content-length"), + headers.dict.keys()))) if content_length: buf += "Content-Length: %s\r\n" % content_length - buf += "Content-Range: bytes %s-%s/%s\r\n" % (0, content_length-1, content_length) - buf += "X-Proxy: Magicproxy (superpower activated)\r\n" + buf += "Content-Range: bytes 0-%s/%s\r\n" % (content_length-1, content_length) + buf += "X-Proxy: Magicproxy; using proxies %s\r\n" % ", ".join(map(lambda host: "%s:%s"%host, self.config.endpoints)) buf += "\r\n" self.channel.push(buf) @@ -250,10 +265,10 @@ class HTTPChannel(asynchat.async_chat): else: MagicHTTPProxyClient(self, url, headers) - def handle_incoming_http_header(self,fetcher,header): + def handle_incoming_http_header(self,fetcher, status_code, header): header.seek(0) headers = httplib.HTTPMessage(header) - buf = "HTTP/1.1 200 OK\r\n" + buf = "HTTP/1.1 %s OK\r\n" % status_code buf += "\r\n".join(map(lambda hdr: "%s: %s" % (hdr,headers.dict[hdr]), headers.dict.keys())) buf += "\r\n\r\n" self.push(buf) |