diff options
Diffstat (limited to 'proxy.py')
| -rwxr-xr-x | proxy.py | 26 |
1 files changed, 19 insertions, 7 deletions
@@ -6,9 +6,9 @@ import cStringIO as StringIO ENDPOINTS = [ - ('10.2.2.11', 8888), -# ('10.3.1.2', 8888), - ('10.1.1.156', 8888), +# ('10.2.2.11', 8888), + ('10.3.1.2', 8888), +# ('10.1.1.156', 8888), ] kB = 1024 @@ -48,7 +48,7 @@ class Fetcher(asynchat.async_chat): self.connect(self.proxy) def __str__(self): - return "<Fetcher proxy=%s url=%s range=%s" % (self.proxy, urlparse.urlunparse(self.url), self.range) + return "<Fetcher proxy=%s host=%s path=%s range=%s" % (self.proxy, self.url.hostname, self.url.path, self.range) def handle_connect (self): print self, "Start" @@ -72,6 +72,16 @@ class Fetcher(asynchat.async_chat): def collect_incoming_data(self, data): if self.state==2: #body length = len(data) + if self.range != (-1,-1) and self.pos + length >= self.range[1]: + #if this request is the first one (whithout Range: ...) then the server + # dont send us our expected range, we must cut it at some point (here) + data=data[:(self.range[1]-self.pos+1)] #XXX explain this + print "cut range=%s pos=%s length=%s => %s" % (self.range, self.pos, length, len(data)) + length = len(data) + if length == 0: + self.reader.handle_incoming_data(self) + self.close() + return self.reader.handle_incoming_data(self, data, length) self.pos += length if self.range != (-1,-1) and self.pos >= self.range[1]: @@ -121,6 +131,7 @@ class MagicHTTPProxyClient(object): #fetcher is done, remove from list self.fetchers = filter(lambda f: f != fetcher, self.fetchers) else: + assert fetcher.pos < fetcher.range[1] or fetcher.range == (-1,-1) heappush(self.blocks, (fetcher.pos, data, length)) if not self.channel.connected: @@ -131,7 +142,7 @@ class MagicHTTPProxyClient(object): and fetcher.range[1] - fetcher.pos < FETCHER_JUMPSTART \ and self.fetch_pos + 1 < self.content_length and self.channel.connected \ and len( filter(lambda f: f.proxy == fetcher.proxy, self.fetchers) ) < 2: - #Start a new fetcher on this line if this fetchers is X-Bytes before finished his job + #Start a new fetcher if this fetcher is X-Bytes before finished his job blocksize = max(int(TIME_SLICE * fetcher.speed()), MIN_BLOCKSIZE) fetch_range = self.next_range(blocksize) print "Start new Fetcher, bs=%s range=%s" % (blocksize,fetch_range) @@ -142,8 +153,9 @@ class MagicHTTPProxyClient(object): item = heappop(self.blocks) buf += item[1] self.write_pos += item[2] - if len(self.blocks)>0: - print "missed: %s => %s" % (self.write_pos, min(self.blocks)[0]) + + if buf == "" and len(self.blocks)>0: + print "search=%s get=%s with length=%s pending=%s" % (self.write_pos, min(self.blocks)[0],min(self.blocks)[2], len(self.blocks)) if buf != "": self.channel.push(buf) |