diff options
Diffstat (limited to 'proxy.py')
| -rw-r--r-- | proxy.py | 86 |
1 files changed, 41 insertions, 45 deletions
@@ -10,20 +10,20 @@ except ImportError: ENDPOINTS = [ ('10.2.2.11', 8888), ('10.3.1.2', 8888), + ('10.1.1.156', 8888), ] #minimum entity size to start a paralel fetch -MINIMAL_SIZE = 524288 -BLOCKSIZE_FETCH = 1048576 - +KB = 1024 +MINIMAL_SIZE = 512 * KB +BLOCKSIZE_FETCH = 2048 * KB class Fetcher(asynchat.async_chat): - def __init__(self, reader, proxy, url, header, range, content_length): - print "Fetcher using proxy=%s for url=%s range=%s content_length=%s" % (proxy, url, range, content_length) + def __init__(self, reader, proxy, url, header, range): self.reader = reader self.proxy = proxy self.url = url + self.header = self.header self.range = range - self.content_length = content_length self.start_time = time.time() @@ -38,28 +38,23 @@ class Fetcher(asynchat.async_chat): #XXXshould include original request headers def handle_connect (self): - print "handle_connect" - self.debug_send("GET http://%s:%s%s HTTP/1.0\r\n" % ( self.url.hostname, self.url.port or 80, self.url.path )) - self.debug_send("Range: bytes=%s-%s\r\n" % (self.range[0], self.range[1])) - self.debug_send("\r\n") - print "sent request" - - def debug_send(self, data): - print "SEND", data - self.send(data) + print self,"handle_connect" + self.send("GET http://%s:%s%s HTTP/1.0\r\n" % ( self.url.hostname, self.url.port or 80, self.url.path )) + #XXX self.header + self.send("Range: bytes=%s-%s\r\n" % (self.range[0], self.range[1])) + self.send("\r\n") def time(self): return self.stop_time - self.start_time def collect_incoming_data(self, data): - if self.state==2:#body - print "read at %s - range_start=%s range_end=%s" % (self.pos, self.range[0], self.range[1]) + if self.state==2: #body self.reader.handle_incoming_data(self.pos, data) self.pos += len(data) if self.pos >= self.range[1]: self.reader.finished() - print "fetcher finished" self.stop_time = time.time() + print self, "finished" self.close_when_done() elif self.state ==1: #header self.http_header += data @@ -70,13 +65,16 @@ class Fetcher(asynchat.async_chat): if self.state == 0: self.state = 1 self.set_terminator("\r\n\r\n") - print "got status line" + #print "got status line" elif self.state == 1: self.state = 2 self.set_terminator(None) - print "header ends now" + print self, "header ends now" self.reader.handle_incoming_http_header(self.http_header) + def __str__(self): + return "<Fetcher proxy=%s url=%s range=%s" % (self.proxy, self.url, self.range) + class MultipleProxyReader(object): def __init__(self, channel, url, header, content_length): print "MultipleProxyReader url=%s content_length=%s" % (url, content_length) @@ -96,7 +94,6 @@ class MultipleProxyReader(object): self.finished() #bad-named XXX def handle_incoming_data(self, pos, data): - print "got body data at pos %s" % pos self.blocks[pos] = data while self.find_next_data(): @@ -108,24 +105,19 @@ class MultipleProxyReader(object): else: self.header_sent = True - self.debug_send("HTTP/1.0 200 OK\r\n") + self.channel.push("HTTP/1.0 200 OK\r\n") + # Sends header from first response headers = httplib.HTTPMessage(StringIO.StringIO(header)) - for key in headers.dict.keys(): - if key in ("content-range", "content-length"): - continue - self.debug_send("%s: %s\r\n" % (key, headers.dict[key])) - self.debug_send("Content-Length: %s" % self.content_length) - self.debug_send("X-Proxy: Magicproxy (superpower activated)\r\n") - self.debug_send("\r\n") - - def debug_send(self, data): - print "SEND", data - self.channel.push(data) + for key in filter(lambda k: k not in ("content-range", "content-length"), headers.dict.keys()): + self.channel.push("%s: %s\r\n" % (key, headers.dict[key])) + self.channel.push("Content-Length: %s" % self.content_length) + self.channel.push("X-Proxy: Magicproxy (superpower activated)\r\n") + self.channel.push("\r\n") def finished(self): self.find_next_data() if self.fetch_pos +1 <self.content_length: - print "start new fetcher" + print self, "start new fetcher" start = self.fetch_pos if self.fetch_pos+BLOCKSIZE_FETCH < self.content_length: self.fetch_pos += BLOCKSIZE_FETCH @@ -136,20 +128,28 @@ class MultipleProxyReader(object): self.next_endpoint = (self.next_endpoint+1) % len(ENDPOINTS) def find_next_data(self): + + if not self.channel.writeable(): # request-side closed the connection + self.channel.close_when_done() + return False + r = False - print "expect at %s" % self.write_pos, self.blocks.keys() + #print "expect at %s" % self.write_pos, self.blocks.keys() if self.write_pos in self.blocks.keys(): r=True data = self.blocks.pop(self.write_pos) self.channel.push(data) self.write_pos += len(data) - print "Send %s bytes" % len(data) + #print "Send %s bytes" % len(data) if self.write_pos + 1 >= self.content_length: - print "done" + print self, "done" self.channel.close_when_done() return r + def __str__(self): + return "<MultipleProxyReader >" + class HTTPChannel(asynchat.async_chat): def __init__(self, server, sock, addr): asynchat.async_chat.__init__(self, sock) @@ -205,16 +205,12 @@ class HTTPProxyServer(asyncore.dispatcher): if len( content_length ) == 0: # no content length given, bypass this request print "missing content-length, bypass" - return self._bypass_request(channel, "GET", url) + self._bypass_request(channel, "GET", url) else: content_length = int(content_length[0][1]) - - #if content_length < MINIMAL_SIZE: - # return self._bypass_request(channel, "GET", url) - - ##XXX - header = (("foo", "bla")) - MultipleProxyReader(channel, url, header, content_length) + ##XXX + header = (("foo", "bla")) + MultipleProxyReader(channel, url, header, content_length) def _bypass_request(self, channel, method, url): print "_bypass request hostname=%s port=%s path=%s" % (url.hostname, url.port or 80, url.path) |