brhute: move to dedicated dir

19 files changed, 99 insertions, 1069 deletions

diff --git a/toys/asynhttp/__init__.py b/toys/asynhttp/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/toys/asynhttp/__init__.py
+++ /dev/null
diff --git a/toys/asynhttp/asynchttp.py b/toys/asynhttp/asynchttp.py
deleted file mode 100644
index b5a2350..0000000
--- a/toys/asynhttp/asynchttp.py
+++ /dev/null
@@ -1,888 +0,0 @@
-#!/usr/bin/env python

-"""Asynchronous HTTP/1.1 client library

-

-This module is an attempt to combine the best features of httplib with

-the scalability of asynchat.

-

-I have pasted as much code as I could from httplib (Python 2.0) because it

-is a well written and widely used interface. This may be a mistake,

-because the behavior of AsynchHTTPConnection is quite different from that of

-httplib.HTTPConnection

-

-contact:

-Doug Fort <dougfort@downright.com>

-Senior Meat Manager

-Downright Software LLC

-http://www.dougfort.com

-

-Bug-fixes and extensions:

-Dhruv Matani <dhruvbird@gmail.com>

-"""

-__author__="""

-Downright Software LLC

-http://www.downright.com

-"""

-__copyright__="""

-Copyright (c) 2001 Downright Software LLC. All Rights Reserved.

-

-Distributed and Licensed under the provisions of the Python Open Source License

-Agreement which is included by reference. (See 'Front Matter' in the latest

-Python documentation)

-

-WARRANTIES

-YOU UNDERSTAND AND AGREE THAT:

-

-a. YOUR USE OF THE PACKAGE IS AT YOUR SOLE RISK.  THE PACKAGE IS PROVIDED ON

-AN 'AS IS' AND 'AS AVAILABLE' BASIS.  DOWNRIGHT EXPRESSLY DISCLAIMS ALL

-WARRANTIES OF ANY KIND, WHETHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED

-TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE

-AND NON-INFRINGEMENT.

-

-b. DOWNRIGHT MAKES NO WARRANTY THAT (1) THE PACKAGE WILL MEET YOUR

-REQUIREMENTS, (2) THE PACKAGE WILL BE UNINTERRUPTED, TIMELY, SECURE, OR

-ERROR-FREE, (3) THE RESULTS THAT MAY BE OBTAINED FROM THE USE OF THE PACKAGE

-WILL BE ACCURATE OR RELIABLE, (4) THE OTHER MATERIAL PURCHASED OR OBTAINED BY

-YOU THROUGH THE PACKAGE WILL MEET YOUR EXPECTATIONS,, AND (5) ANY ERRORS IN

-THE PACKAGE WILL BE CORRECTED.

-

-c. ANY MATERIALS DOWNLOADED OR OTHERWISE OBTAINED THROUGH THE USE OF THE

-PACKAGE IS DONE AT YOUR OWN DISCRETION AND RISK AND THAT YOU WILL BE SOLELY

-RESPONSIBLE FOR ANY DAMAGE TO YOUR COMPUTER SYSTEM OR LOSS OF DATA THAT

-RESULTS FROM THE DOWNLOAD OF ANY SUCH MATERIAL.

-

-d. NO ADVICE OR INFORMATION, WHETHER ORAL OR WRITTEN, OBTAINED BY YOU FROM

-DOWNRIGHT OR THROUGH OR FROM THE PACKAGE SHALL CREATE ANY WARRANTY NOT

-EXPRESSLY STATED IN THE TOS.

-

-LIMITATION OF LIABILITY

-YOU EXPRESSLY UNDERSTAND AND AGREE THAT DOWNRIGHT SHALL NOT BE LIABLE FOR ANY

-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL OR EXEMPLARY DAMAGES,

-INCLUDING BUT NOT LIMITED TO, DAMAGES FOR LOSS OF PROFITS, GOODWILL, USE,

-DATA OR OTHER INTANGIBLE LOSSES (EVEN IF DOWNRIGHT HAS BEEN ADVISED OF SUCH

-DAMAGES), RESULTING FROM:

-(1) THE USE OR THE INABILITY TO USE THE PACKAGE;

-(2) THE COST OF PROCUREMENT OF SUBSTITUTE GOODS AND SERVICES RESULTING FROM

-ANY GOODS, DATA, INFORMATION OR SERVICES PURCHASED OR OBTAINED OR MESSAGES

-RECEIVED OR TRANSACTIONS ENTERED INTO THROUGH OR FROM THE PACKAGE;

-(3) UNAUTHORIZED ACCESS TO OR ALTERATION OF YOUR TRANSMISSIONS OR DATA;

-(4) STATEMENTS OF CONDUCT OF ANY THIRD PARTY ON THE PACKAGE; OR

-(5) ANY OTHER MATTER RELATING TO THE PACKAGE.

-"""

-__version__="0.20"

-

-import sys

-import asynchat

-import asyncore

-import socket

-import time

-import string

-import cStringIO

-import mimetools

-

-HTTP_PORT = 80

-HTTPS_PORT = 443

-

-

-class AsyncHTTPResponse:

-    """

-    This class attempts to mimic HTTPResponse from httplib.

-    The major difference is that it is NOT DYNAMIC:

-    All the reading has already been done

-    """

-    def __init__(self, fp, debuglevel=0):

-        """

-        This constructor builds everything in the response

-        object except the body.  It expects a file object

-        containing the header text returned by the server

-        """

-        self.debuglevel = debuglevel

-

-        # we're expecting something like 'HTTP/1.1 200 OK'

-        self._replyline = fp.readline()

-        if self.debuglevel > 0:

-            print "reply: %s" % (self._replyline)

-

-        replylist = string.split(self._replyline, None, 2)

-

-        if len(replylist) == 3:

-            version, status, reason = replylist

-        elif len(replylist) == 2:

-            version, status = replylist

-            reason = ""

-        else:

-            raise BadStatusLine(self._replyline, name=str(self))

-

-        if version[:5] != 'HTTP/':

-            raise BadStatusLine(self._replyline, name=str(self))

-

-        try:

-            self.status = int(status)

-        except:

-            raise BadStatusLine(self._replyline, name=str(self))

-

-        self.reason = string.strip(reason)

-

-        if version == 'HTTP/1.0':

-            self.version = 10

-        elif version.startswith('HTTP/1.'):

-            self.version = 11    # use HTTP/1.1 code for HTTP/1.x where x>=1

-        else:

-            raise UnknownProtocol(self._replyline, name=str(self))

-

-        self.msg = mimetools.Message(fp, 0)

-        if self.debuglevel > 0:

-            for hdr in self.msg.headers:

-                print "header: %s" %  (string.strip(hdr))

-

-        self.body = None

-

-    def __str__(self):

-        return "AsyncHTTPResponse %s" % (self._replyline)

-

-    def getheader(self, name, default=None):

-        if self.msg is None:

-            raise ResponseNotReady(name=str(self))

-        return self.msg.getheader(name, default)

-

-    def getbody(self):

-        if self.body is None:

-            raise ResponseNotReady(name=str(self))

-        return self.body

-

-_CHUNK_REQUEST_SIZE = 8192

-

-_STATE_IDLE = "asynchttp._STATE_IDLE"

-_STATE_CONNECTING = "asynchttp._STATE_CONNECTING"

-_STATE_ACTIVE = "asynchttp._STATE_ACTIVE"

-_STATE_ACCEPTING_HEADERS = "asynchttp._STATE_ACCEPTING_HEADERS"

-_STATE_REQUESTING_BODY = "asynchttp._STATE_REQUESTING_BODY"

-_STATE_CHUNK_START = "asynchttp._STATE_CHUNK_START"

-_STATE_CHUNK_BODY = "asynchttp._STATE_CHUNK_BODY"

-_STATE_CHUNK_RESIDUE = "asynchttp._STATE_CHUNK_RESIDUE"

-

-class AsyncHTTPConnection(asynchat.async_chat):

-

-    _http_vsn = 11

-    _http_vsn_str = 'HTTP/1.1'

-

-    response_class = AsyncHTTPResponse

-    default_port = HTTP_PORT

-    auto_open = 0

-    debuglevel = 0

-

-    def __init__(self, host=None, port=None):

-        asynchat.async_chat.__init__(self)

-        self.socket = None

-

-        # overload asynchat.found_terminator with the function

-        # appropriate for each state

-        self._TERMINATOR_MAP = {

-            _STATE_IDLE : self._no_action,

-            _STATE_CONNECTING :  self._no_action,

-            _STATE_ACTIVE:  self._no_action,

-            _STATE_ACCEPTING_HEADERS:  self._header_data,

-            _STATE_REQUESTING_BODY : self._body_data,

-            _STATE_CHUNK_START : self._chunk_start_data,

-            _STATE_CHUNK_BODY : self._chunk_body_data,

-            _STATE_CHUNK_RESIDUE : self._chunk_residue_data

-            }

-

-        self.__state = None

-        self.__set_state(_STATE_IDLE)

-

-        # we accumulate headers in a dictionary so the

-        # caller can write over the headers we supply

-        # (or their own headers if  they want)

-        self._headerdict = None

-        self._requestfp = None

-        self._responsefp = None

-        self._chunkfp = None

-

-        self._set_hostport(host, port)

-

-        self._willclose = 0

-

-    def _set_hostport(self, host, port):

-        if host and port is None:

-            i = string.find(host, ':')

-            if i >= 0:

-                port = int(host[i+1:])

-                host = host[:i]

-            else:

-                port = self.default_port

-

-        self.host = host

-        self.port = port

-

-    def set_debuglevel(self, level):

-        self.debuglevel = level

-

-    def connect(self):

-        """

-        Connect to the host and port specified in __init__.

-        Add ourselves to the asyncore polling group

-        """

-        self.__set_state(_STATE_CONNECTING)

-        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)

-        if self.debuglevel > 0:

-            print "connecting: (%s, %s)" % (self.host, self.port)

-

-        asyncore.dispatcher.connect(self, (self.host, self.port))

-

-    def close(self):

-        """

-        Close the connection to the HTTP server.

-        And remove ourselves from the asyncore polling group

-        """

-        if self.debuglevel > 0:

-            print "asynchttp.close() (%s, %s)" % (self.host, self.port)

-

-        self.connected = 0

-

-        if self.socket:

-            asynchat.async_chat.close(self)

-

-        self._set_hostport(None, None)

-

-    def send_entity(self, str):

-        """

-        Send `str' to the server.

-        Actually, we  just append str to the block of text to  be sent

-        to the server when getresponse is called.

-

-        Note:  the name was changed from httplib's 'HTTPConnection.send()'

-        because it conflicts with asynchat

-        """

-        if self.debuglevel > 0:

-            print "send_entity %s"

-

-        self._requestfp.write(str)

-

-    def putrequest(self, method, url):

-        """Send a request to the server.

-

-        `method' specifies an HTTP request method, e.g. 'GET'.

-        `url' specifies the object being requested, e.g. '/index.html'.

-

-        This function actually only starts accumulating the request:

-        nothing gets sent to the server until getresponse() is called.

-        """

-        if self.debuglevel > 0:

-            print "putrequest %s %s" % (method, url)

-

-        # Dhruv: Commented out the check below since it interferes with

-        # HTTP pipelining

-        """if not self.__state is _STATE_ACTIVE:

-            raise RequestNotReady(

-                "Invalid putrequest() %s" % (self.__state),

-                name=str(self)

-                )

-        """

-

-        self._requestfp = cStringIO.StringIO()

-

-        if not url:

-            url = '/'

-        self._requestfp.write(

-            '%s %s %s\r\n' % (method, url, self._http_vsn_str)

-            )

-

-        self._headerdict = {}

-

-        if self._http_vsn == 11:

-            # Issue some standard headers for better HTTP/1.1 compliance

-

-            # this header is issued *only* for HTTP/1.1 connections. more

-            # specifically, this means it is only issued when the client uses

-            # the new HTTPConnection() class. backwards-compat clients will

-            # be using HTTP/1.0 and those clients may be issuing this header

-            # themselves. we should NOT issue it twice; some web servers (such

-            # as Apache) barf when they see two Host: headers

-            self.putheader('Host', self.host)

-

-            # note: we are assuming that clients will not attempt to set these

-            #       headers since *this* library must deal with the

-            #       consequences. this also means that when the supporting

-            #       libraries are updated to recognize other forms, then this

-            #       code should be changed (removed or updated).

-

-            # we only want a Content-Encoding of "identity" since we don't

-            # support encodings such as x-gzip or x-deflate.

-            self.putheader('Accept-Encoding', 'identity')

-

-            # we can accept "chunked" Transfer-Encodings, but no others

-            # NOTE: no TE header implies *only* "chunked"

-            #self.putheader('TE', 'chunked')

-

-            # if TE is supplied in the header, then it must appear in a

-            # Connection header.

-            #self.putheader('Connection', 'TE')

-

-    def putheader(self, header, value):

-        """

-        Send a request header line to the server.

-

-        For example: h.putheader('Accept', 'text/html')

-        We don't actually send the header here, we stick it

-        in a dictionary, to be sent when getresponse() is

-        called.  If you call putheader() with a duplicate

-        key, it will wipe out the existing entry.

-        """

-        if self.debuglevel > 0:

-            print "putheader %s: %s" % (header, value)

-

-        self._headerdict[header] = value

-

-    def endheaders(self):

-        """

-        Indicate that the last header line has been sent to the server.

-        Actually, we just copy the header dictionary into the request

-        stream to be sent when getresponse() is called.

-        """

-        if self.debuglevel > 0:

-            print "endheaders"

-

-        for header, value in self._headerdict.items():

-            self._requestfp.write(

-                '%s: %s\r\n' % (header, value)

-                )

-        # store a blank line to indicate end of headers

-        self._requestfp.write('\r\n')

-

-    def request(self, method, url, body=None, headers={}):

-        """

-        Send a complete request to the server.

-        """

-        if self.debuglevel > 0:

-            print "request"

-

-        self._send_request(method, url, body, headers)

-

-    def _send_request(self, method, url, body, headers):

-        if self.debuglevel > 0:

-            print "_send_request"

-

-        self.putrequest(method, url)

-

-        if body:

-            self.putheader('Content-Length', str(len(body)))

-

-        for hdr, value in headers.items():

-            self.putheader(hdr, value)

-

-        self.endheaders()

-

-        if body:

-            self.send_entity(body)

-

-    def push_request(self):

-        """

-        Push a single HTTP request to the server. Multiple requests can be

-        pushed to the server at the same time. This is called HTTP request

-        pipelining

-        """

-        self.push(self._requestfp.getvalue())

-        self._requestfp = None

-

-    def pop_response(self):

-        """

-        Fetch a single response to the first (in time) unanswered request.

-        The response will be available in the response variable. Save it

-        elsewhere and call pop_response again to fetch the next response

-        from the server

-        """

-        self.__set_state(_STATE_ACCEPTING_HEADERS)

-        self.set_terminator("\r\n\r\n")

-        self._responsefp = cStringIO.StringIO()

-

-    def getresponse(self):

-        """

-        Get the response from the server.

-        This  actually starts the process of sending the request

-        to the server.  The response will be delivered in handle_response

-        """

-        self.__set_state(_STATE_ACCEPTING_HEADERS)

-

-        self.push(self._requestfp.getvalue())

-

-        self._requestfp = None

-

-        # exit this state on a blank line

-        self.set_terminator("\r\n\r\n")

-

-        self._responsefp = cStringIO.StringIO()

-

-    def handle_connect(self):

-        """

-        Notification from asyncore that we are connected

-        """

-        self.__set_state(_STATE_ACTIVE)

-        if self.debuglevel > 0:

-            print "connected: (%s, %s)" % (self.host, self.port)

-

-    def handle_close(self):

-        """

-        Notification from asyncore that the server has closed

-        its end of the connection.

-        If auto_open is TRUE, we will attempt to reopen the

-        connection.

-        """

-        if self.debuglevel > 0:

-            print "closed by server: (%s, %s) %s" % (

-                self.host, self.port, self.__state

-                )

-

-        # 2001-03-14 djf If the server closed the connection while we're

-        # requesting body data, it  may just be trying to tell us that

-        # we're done

-        if self.__state in [

-            _STATE_REQUESTING_BODY,

-            _STATE_CHUNK_BODY,

-            _STATE_CHUNK_RESIDUE

-            ]:

-            self.found_terminator()

-            return

-

-        asynchat.async_chat.handle_close(self)

-

-        # if auto_open, attempt to reopen the connection

-        if AsyncHTTPConnection.auto_open and self.host:

-            self.connect()

-

-    def was_handle_read (self):

-

-        try:

-            data = self.recv (self.ac_in_buffer_size)

-        except socket.error, why:

-            self.handle_error()

-            return

-

-        self.ac_in_buffer = self.ac_in_buffer + data

-

-        # Continue to search for self.terminator in self.ac_in_buffer,

-        # while calling self.collect_incoming_data.  The while loop

-        # is necessary because we might read several data+terminator

-        # combos with a single recv(1024).

-

-        while self.ac_in_buffer:

-            lb = len(self.ac_in_buffer)

-            terminator = self.get_terminator()

-            if terminator is None:

-                # no terminator, collect it all

-                self.collect_incoming_data (self.ac_in_buffer)

-                self.ac_in_buffer = ''

-            elif type(terminator) == type(0):

-                # numeric terminator

-                n = terminator

-                if lb < n:

-                    self.collect_incoming_data (self.ac_in_buffer)

-                    self.ac_in_buffer = ''

-                    self.terminator = self.terminator - lb

-                else: # give back the whole pizza

-                    self.collect_incoming_data (self.ac_in_buffer)

-                    self.ac_in_buffer = ''

-                    self.terminator = 0

-                    self.found_terminator()

-            else:

-                # 3 cases:

-                # 1) end of buffer matches terminator exactly:

-                #    collect data, transition

-                # 2) end of buffer matches some prefix:

-                #    collect data to the prefix

-                # 3) end of buffer does not match any prefix:

-                #    collect data

-                terminator_len = len(terminator)

-                index = string.find (self.ac_in_buffer, terminator)

-                if index != -1:

-                    # we found the terminator

-                    if index > 0:

-                        # don't bother reporting the empty string (source of subtle bugs)

-                        self.collect_incoming_data (self.ac_in_buffer[:index])

-                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]

-                    # This does the Right Thing if the terminator is changed here.

-                    self.found_terminator()

-                else:

-                    # check for a prefix of the terminator

-                    index = asynchat.find_prefix_at_end (self.ac_in_buffer, terminator)

-                    if index:

-                        if index != lb:

-                            # we found a prefix, collect up to the prefix

-                            self.collect_incoming_data (self.ac_in_buffer[:-index])

-                            self.ac_in_buffer = self.ac_in_buffer[-index:]

-                            break

-                        else:

-                            # no prefix, collect it all

-                            self.collect_incoming_data (self.ac_in_buffer)

-                            self.ac_in_buffer = ''

-

-

-    def handle_error(self):

-        """

-        Overload asyncore's exception handling

-        """

-        self.__set_state(_STATE_IDLE)

-        asyncore.dispatcher.handle_error(self)

-

-    def collect_incoming_data(self, data):

-        """

-        asynchat calls this with data as it comes in

-        """

-        if not self._responsefp:

-            raise UnexpectedData(

-                "%s '%s' '%s' '%s'" % (

-                self.__state,

-                data,

-                self.get_terminator(),

-                self.ac_in_buffer

-                ), name=str(self))

-

-        self._responsefp.write(data)

-

-    def _no_action(self):

-        """

-        overload asynchat.found_terminator

-        This function will only be called when someone is badly confused

-        """

-        raise UnexpectedTerminator(

-            "%s '%s'" % (self.__state, self.get_terminator()),

-            name=str(self)

-            )

-

-    def _header_data(self):

-        """

-        overload asynchat.found_terminator for

-        _STATE_ACCEPTING_HEADERS

-        We assume that we have hit the blank line terminator after the

-        HTTP response headers.

-        """

-        self._responsefp.seek(0)

-        self.response = self.response_class(

-            self._responsefp,

-            self.debuglevel

-            )

-

-        self._willclose = string.lower(

-            self.response.getheader("connection", "")

-            ) == "close"

-

-        transferencoding = string.lower(

-            self.response.getheader("transfer-encoding", "")

-            )

-

-        # set up for getting the body

-        self._responsefp = cStringIO.StringIO()

-

-        if transferencoding:

-            if transferencoding == "chunked":

-                self._chunkfp = cStringIO.StringIO()

-                self.set_terminator("\r\n")

-                self.__set_state(_STATE_CHUNK_START)

-                return

-

-            raise UnknownTransferEncoding(

-                self.response.getheader("transfer-encoding", ""),

-                name=str(self)

-                )

-

-        contentlengthstr = self.response.getheader(

-            "content-length", None

-            )

-        if contentlengthstr:

-            contentlength = int(contentlengthstr)

-        else:

-            contentlength = None

-

-        self.set_terminator(contentlength)

-        self.__set_state(_STATE_REQUESTING_BODY)

-

-    def _body_data(self):

-        """

-        overload asynchat.found_terminator for

-        _STATE_REQUESTING_BODY

-        We assume that we have the full body text

-        """

-        self.response.body = self._responsefp.getvalue()

-        self._responsefp = None

-

-        if self._willclose:

-            self.close()

-

-        self.__set_state(_STATE_ACTIVE)

-

-        # hand off the response object to the child class

-        self.handle_response()

-

-    def _get_chunk_size(self):

-        """

-        Assume that chunkbuffer contains some text, begining with

-        a line containing the chunk size in hex.

-        """

-        # 2001-03-26 djf -- kludge alert! We shouldn't have to lstrip

-        # here, but sometimes we get extra whitespace

-        splitlist = self._chunkbuffer.lstrip().split("\r\n",1)

-        if len(splitlist) == 1:

-            chunkline, self._chunkbuffer = splitlist[0], ''

-        else:

-            chunkline, self._chunkbuffer = splitlist

-

-        i = string.find(chunkline, ';')

-        if i >= 0:

-            chunkline = chunkline[:i]    # strip chunk-extensions

-

-        try:

-            chunksize = string.atoi(chunkline, 16)

-        except:

-            raise InvalidChunk(

-                "Can't compute chunk size from '%s' '%s'" % (

-                chunkline, self._chunkbuffer

-                ))

-

-        if self.debuglevel > 0:

-            print "chunksize = '%d" % (chunksize)

-

-        return chunksize

-

-    def _chunk_start_data(self):

-        """

-        overload asynchat.found_terminator for

-        _STATE_CHUNKED_START

-        Assumes we got a hit on terminator '\r\n'

-        """

-        self._chunkbuffer = self._responsefp.getvalue()

-        self._chunksize = self._get_chunk_size()

-        if self._chunksize == 0:

-            if self.debuglevel > 0:

-                print "0 size Chunk: ending chunk processing"

-            self.response.body = self._chunkfp.getvalue()

-            self._chunkfp = None

-            self.set_terminator("\r\n") # was: \r\n\r\n

-            self._responsefp = cStringIO.StringIO()

-            self.__set_state(_STATE_CHUNK_RESIDUE)

-            return

-

-        self.set_terminator(self._chunksize+2)

-        self._responsefp = cStringIO.StringIO()

-        self.__set_state(_STATE_CHUNK_BODY)

-

-    def _chunk_body_data(self):

-        """

-        overload asynchat.found_terminator for

-        _STATE_CHUNK_BODY

-        """

-        self._chunkbuffer += self._responsefp.getvalue()

-

-        while self._chunkbuffer:

-            chunk_plus_crlf_size = self._chunksize+2

-            if len(self._chunkbuffer) > chunk_plus_crlf_size:

-                # DHRUV: WAS: chunkbody = self._chunkbuffer[:chunk_plus_crlf_size]

-                chunkbody = self._chunkbuffer[:self._chunksize]

-                self._chunkbuffer = self._chunkbuffer[chunk_plus_crlf_size:]

-                self._chunkbuffer = self._chunkbuffer.lstrip()

-            else:

-                # DHRUV: WAS: chunkbody = self._chunkbuffer

-                chunkbody = self._chunkbuffer[:-2]

-                self._chunkbuffer = ''

-

-            self._chunkfp.write(chunkbody)

-

-            if not self._chunkbuffer:

-                break

-

-            print "SSSSSSSSSSSSXXXXXXXXXXXX"

-            print "LEFT: " + self._chunkbuffer

-            sys.exit(1)

-            # Dhruv: If we have some more data, we ignore it.

-            #if we have some text left over, we hope it's another chunk,

-            # but if it doesn't contain a newline, it is insufficient

-            if self._chunkbuffer.find("\r\n") < 0:

-                self._responsefp = cStringIO.StringIO()

-                self.set_terminator("\r\n")

-                self.__set_state(_STATE_CHUNK_START)

-                return

-

-            self._chunksize = self._get_chunk_size()

-            if self._chunksize == 0:

-                if self.debuglevel > 0:

-                    print "0 size Chunk: ending chunk processing"

-                self.response.body = self._chunkfp.getvalue()

-                self._chunkfp = None

-

-                # if there's still something in the buffer,

-                # assume it's  the chunk residue (probably just

-                # '\r\n'

-                if self._chunkbuffer:

-                    self._chunkbuffer = "" # discard the residue

-                    self.__set_state(_STATE_ACTIVE)

-

-                    if self._willclose:

-                        self.close()

-

-                    # hand off the response object to the child class

-                    self.handle_response()

-                    return

-

-                # we've handled the whole chunk, but the server could

-                # send entity headers. It should  at least send

-                # a final '\r\n'

-                self.set_terminator("\r\n\r\n")

-                self._responsefp = cStringIO.StringIO()

-                self.__set_state(_STATE_CHUNK_RESIDUE)

-                return

-

-            # We have a nonzero chunksize, if we have less than

-            # the specified number of bytes in the buffer, we need

-            # to  read some more

-            chunk_plus_crlf_size = self._chunksize+2

-            bufsize = len(self._chunkbuffer)

-            if bufsize < chunk_plus_crlf_size:

-                self.set_terminator(chunk_plus_crlf_size - bufsize)

-                self._responsefp = cStringIO.StringIO()

-                self.__set_state(_STATE_CHUNK_BODY)

-                return

-

-            # if we made it this far, we should have a chunk size and

-            # at least enough in the buffer to satisfy it. So we loop

-            # back to the top of the while.

-

-        # we don't have any text left over, but we haven't hit a

-        # zero chunk. See if the server will give us another line

-        self._responsefp = cStringIO.StringIO()

-        self.set_terminator("\r\n")

-        self.__set_state(_STATE_CHUNK_START)

-

-    def _chunk_residue_data(self):

-        """

-        overload asynchat.found_terminator for

-        _STATE_CHUNK_RESIDUE

-        """

-        residue = string.strip(self._responsefp.getvalue())

-        if self.debuglevel > 0 and residue:

-            print "chunk residue '%s'" % (residue)

-

-        self._responsefp = None

-

-        if self._willclose:

-            self.close()

-

-        self.__set_state(_STATE_ACTIVE)

-

-        # hand off the response object to the child class

-        self.handle_response()

-

-    def handle_response(self):

-        """

-        This is an abstract function, the  user MUST overload it

-        """

-        raise HandleResponse(

-            "Call to AsyncHTTPConnection.handle_response", name=str(self)

-            )

-

-    def __set_state(self, next_state):

-        """

-        Change state be setting _found_terminator

-        """

-        if self.debuglevel > 0:

-            print "%s to %s" % (self.__state, next_state)

-        self.__state = next_state

-        self.found_terminator = self._TERMINATOR_MAP[self.__state]

-

-

-class AsyncHTTPException(Exception):

-    def __init__(self, message="", name=""):

-        self._message = message

-        self._name = name

-

-    def __str__(self):

-        return "%s %s" % (self._name, self._message)

-

-class NotConnected(AsyncHTTPException):

-    pass

-

-class UnknownProtocol(AsyncHTTPException):

-    pass

-

-class UnknownTransferEncoding(AsyncHTTPException):

-    pass

-

-class BadStatusLine(AsyncHTTPException):

-    pass

-

-class ImproperConnectionState(AsyncHTTPException):

-    pass

-

-class RequestNotReady(ImproperConnectionState):

-    pass

-

-class ResponseNotReady(ImproperConnectionState):

-    pass

-

-class HandleResponse(ImproperConnectionState):

-    pass

-

-class UnexpectedData(AsyncHTTPException):

-    pass

-

-class UnexpectedTerminator(AsyncHTTPException):

-    pass

-

-class InvalidChunk(AsyncHTTPException):

-    pass

-

-class __test_AsyncHTTPConnection(AsyncHTTPConnection):

-    def __init__(self, host, port, url):

-        AsyncHTTPConnection.__init__(

-            self, host, port

-            )

-        self._url = url

-

-

-    def handle_expt(self):

-        print "Hello"

-        self.close()

-        # sys.exit(1)

-

-    def handle_response(self):

-        self.close()

-

-    def handle_connect(self):

-        print "__test_AsyncHTTPConnection.handle_connect"

-        AsyncHTTPConnection.handle_connect(self)

-        self.putrequest("GET", self._url)

-        self.endheaders()

-        self.getresponse()

-

-if __name__ == "__main__":

-    """

-    Code for command line testing

-    """

-    if len(sys.argv) < 4:

-        print "Usage:  asynchttp.py <host> <port> <request>"

-        sys.exit(-1)

-

-    tester = __test_AsyncHTTPConnection(

-        sys.argv[1],

-        int(sys.argv[2]),

-        sys.argv[3]

-        )

-    tester.set_debuglevel(1)

-    tester.connect()

-

-    asyncore.loop()

-

-    if not hasattr(tester, "response"):

-        print "No response"

-        sys.exit(-1)

-

-    print "results %s %d %s" % (

-        tester.response.version,

-        tester.response.status,

-        tester.response.reason

-        )

-

-    print "headers:"

-    for hdr in tester.response.msg.headers:

-        print "%s" %  (string.strip(hdr))

-

-    if tester.response.status == 200:

-        print "body:"

-        print tester.response.body

diff --git a/toys/asynhttp/http_evented.py b/toys/asynhttp/http_evented.py
deleted file mode 100644
index b5dbf34..0000000
--- a/toys/asynhttp/http_evented.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""Event based HTTP/1.1 client library

-

-This module is an attempt to create a true asynchronous event based 

-(javascript like) HTTP request-response interface. It is built up on

-the asynchttp client interface.

-

-contact:

-Dhruv Matani <dhruvbird@gmail.com>

-"""

-__author__="""

-Dhruv Matani

-"""

-__copyright__="""

-Copyright (c) 2010 Dhruv Matani.

-

-Distributed and Licensed under the provisions of the Python Open Source License

-Agreement which is included by reference. (See 'Front Matter' in the latest

-Python documentation)

-

-WARRANTIES

-YOU UNDERSTAND AND AGREE THAT:

-

-a. YOUR USE OF THE PACKAGE IS AT YOUR SOLE RISK.  THE PACKAGE IS PROVIDED ON

-AN 'AS IS' AND 'AS AVAILABLE' BASIS.  DOWNRIGHT EXPRESSLY DISCLAIMS ALL

-WARRANTIES OF ANY KIND, WHETHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED

-TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE

-AND NON-INFRINGEMENT.

-

-b. DOWNRIGHT MAKES NO WARRANTY THAT (1) THE PACKAGE WILL MEET YOUR

-REQUIREMENTS, (2) THE PACKAGE WILL BE UNINTERRUPTED, TIMELY, SECURE, OR

-ERROR-FREE, (3) THE RESULTS THAT MAY BE OBTAINED FROM THE USE OF THE PACKAGE

-WILL BE ACCURATE OR RELIABLE, (4) THE OTHER MATERIAL PURCHASED OR OBTAINED BY

-YOU THROUGH THE PACKAGE WILL MEET YOUR EXPECTATIONS,, AND (5) ANY ERRORS IN

-THE PACKAGE WILL BE CORRECTED.

-

-c. ANY MATERIALS DOWNLOADED OR OTHERWISE OBTAINED THROUGH THE USE OF THE

-PACKAGE IS DONE AT YOUR OWN DISCRETION AND RISK AND THAT YOU WILL BE SOLELY

-RESPONSIBLE FOR ANY DAMAGE TO YOUR COMPUTER SYSTEM OR LOSS OF DATA THAT

-RESULTS FROM THE DOWNLOAD OF ANY SUCH MATERIAL.

-

-d. NO ADVICE OR INFORMATION, WHETHER ORAL OR WRITTEN, OBTAINED BY YOU FROM

-DOWNRIGHT OR THROUGH OR FROM THE PACKAGE SHALL CREATE ANY WARRANTY NOT

-EXPRESSLY STATED IN THE TOS.

-

-LIMITATION OF LIABILITY

-YOU EXPRESSLY UNDERSTAND AND AGREE THAT DOWNRIGHT SHALL NOT BE LIABLE FOR ANY

-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL OR EXEMPLARY DAMAGES,

-INCLUDING BUT NOT LIMITED TO, DAMAGES FOR LOSS OF PROFITS, GOODWILL, USE,

-DATA OR OTHER INTANGIBLE LOSSES (EVEN IF DOWNRIGHT HAS BEEN ADVISED OF SUCH

-DAMAGES), RESULTING FROM:

-(1) THE USE OR THE INABILITY TO USE THE PACKAGE;

-(2) THE COST OF PROCUREMENT OF SUBSTITUTE GOODS AND SERVICES RESULTING FROM

-ANY GOODS, DATA, INFORMATION OR SERVICES PURCHASED OR OBTAINED OR MESSAGES

-RECEIVED OR TRANSACTIONS ENTERED INTO THROUGH OR FROM THE PACKAGE;

-(3) UNAUTHORIZED ACCESS TO OR ALTERATION OF YOUR TRANSMISSIONS OR DATA;

-(4) STATEMENTS OF CONDUCT OF ANY THIRD PARTY ON THE PACKAGE; OR

-(5) ANY OTHER MATTER RELATING TO THE PACKAGE.

-"""

-

-from asynchttp import AsyncHTTPConnection

-import collections

-

-STATE_CONNECTING   = 1

-STATE_CONNECTED    = 2

-STATE_DISCONNECTED = 3

-

-def call_if_not_none_and_callable(o, **kwargs):

-    if o is not None and callable(o):

-        o(**kwargs)

-

-class http_evented(AsyncHTTPConnection, object):

-    """

-    This is an event based async HTTP client. It lets you register

-    events which will be called whenever an HTTP request is completed.

-

-    It sort of mimics the javascript way of making ajax calls, which

-    I have started to like for various reasons -- the last of which

-    is efficiency though ;)

-

-    Use function parameter binding while using this module to get the

-    maximum bang for your buck :D

-

-    DO NOT use request pipelining with unordered responses along with

-    this class if you are expecting your handlers to be called. You may

-    however use standard HTTP request pipelining in which responses are

-    guaranteed to be returned in the same order as the requests are made

-

-    However, if you are handling re-ordering of responses at a higher

-    layer, then you may use it as you feel free. Make sure that that you

-    are prepared to handle the calling of the event handlers in ANY order

-    when the response is received from the server. You can typically

-    handle this by always registering the same function for every request

-    that you make

-    """

-    def __init__(self, host_and_port, onConnected=None,

-                 onClose=None, onException=None):

-        self._connection_state = STATE_DISCONNECTED

-        self._eventHandlers    = collections.deque()

-        self._onClose          = onClose

-        self._onException      = onException

-        self._onConnected      = None

-        self.reconnect(host_and_port, onConnected)

-

-    def reconnect(self, host_and_port, onReconnect=None):

-        """

-        [Re]connect to the HTTP end point

-        """

-        if self._connection_state != STATE_CONNECTING and self._connection_state == STATE_DISCONNECTED:

-            host, port = host_and_port

-            self._connection_state = STATE_CONNECTING

-            self._onConnected = onReconnect

-            AsyncHTTPConnection.__init__(self, host, port)

-            self.connect()

-

-    def handle_response(self):

-        """

-        Called when a response from the server is received

-        """

-        call_if_not_none_and_callable(self._eventHandlers.popleft(),

-                                      response=self.response)

-

-    def handle_connect(self):

-        """

-        Called when the connection to the HTTP end point succeeds

-        """

-        print "http_evented::handle_connect"

-        self._connection_state = STATE_CONNECTED

-        super(http_evented, self).handle_connect()

-        call_if_not_none_and_callable(self._onConnected)

-

-    def handle_close(self):

-        """

-        Called when the connection is closed from the server end

-        """

-        self._connection_state = STATE_DISCONNECTED

-        super(http_evented, self).handle_close()

-        self._fail_all_pending_event_handlers()

-        call_if_not_none_and_callable(self._onClose)

-

-    def handle_error(self):

-        super(http_evented, self).handle_error()

-        self._perform_on_error_handling()

-

-    def handle_expt(self):

-        """

-        Called in case an exception is thrown while executing code.

-        This can also happen due to disconnection if the remote

-        HTTP end point goes down

-        """

-        self._perform_on_error_handling()

-

-    def _perform_on_error_handling(self):

-        self._connection_state = STATE_DISCONNECTED

-        self._fail_all_pending_event_handlers()

-        call_if_not_none_and_callable(self._onException)

-

-    def push_HTTP_request(self, method, url, body, headers, callback=None):

-        """

-        Just PUSH the request on to the request queue. It will be sent

-        ONLY when pop_response() is called

-        """

-        self.request(method, url, body, headers)

-        self.push_request()

-        self._eventHandlers.append(callback)

-

-    def make_HTTP_request(self, method, url, body, headers, callback=None):

-        """

-        Make an HTTP request to the other HTTP end point. The callable

-        'callback' will be called with either no parameter OR a single

-        parameter named 'response' which will hold the HTTP response

-        object. If there is an error or the response could NOT be

-        processed for some reason, then response=None is passed to the

-        callback function

-        """

-        self.push_HTTP_request(method, url, body, headers, callback)

-        self.pop_response()

-

-    def _fail_all_pending_event_handlers(self):

-        for eh in self._eventHandlers:

-            call_if_not_none_and_callable(eh, response=None)

-        self._eventHandlers.clear()

diff --git a/toys/brhute.py b/toys/brhute/brhute.py
index 851d742..851d742 100644
--- a/toys/brhute.py
+++ b/toys/brhute/brhute.py
diff --git a/toys/brhute_multitprocessing_broken.py b/toys/brhute/brhute_multitprocessing_broken.py
index fb7f19d..fb7f19d 100644
--- a/toys/brhute_multitprocessing_broken.py
+++ b/toys/brhute/brhute_multitprocessing_broken.py
diff --git a/toys/brhute_threaded.py b/toys/brhute/brhute_threaded.py
index 22ab2bb..22ab2bb 100644
--- a/toys/brhute_threaded.py
+++ b/toys/brhute/brhute_threaded.py
diff --git a/toys/brhute_twisted.py b/toys/brhute/brhute_twisted.py
index 22ab2bb..22ab2bb 100644
--- a/toys/brhute_twisted.py
+++ b/toys/brhute/brhute_twisted.py
diff --git a/toys/brhute/contrib/httplib_pipelining_orig.py b/toys/brhute/contrib/httplib_pipelining_orig.py
new file mode 100644
index 0000000..75e9aad
--- /dev/null
+++ b/toys/brhute/contrib/httplib_pipelining_orig.py
@@ -0,0 +1,99 @@
+## {{{ http://code.activestate.com/recipes/576673/ (r5)
+from httplib import HTTPConnection, _CS_IDLE
+import urlparse
+
+def pipeline(domain,pages,max_out_bound=4,debuglevel=0):
+    pagecount = len(pages)
+    conn = HTTPConnection(domain)
+    conn.set_debuglevel(debuglevel)
+    respobjs = [None]*pagecount
+    finished = [False]*pagecount
+    data = [None]*pagecount
+    headers = {'Host':domain,'Content-Length':0,'Connection':'Keep-Alive'}
+
+    while not all(finished):
+        # Send 
+        out_bound = 0
+        for i,page in enumerate(pages):
+            if out_bound >= max_out_bound:
+                break
+            elif page and not finished[i] and respobjs[i] is None:
+                if debuglevel > 0:
+                    print 'Sending request for %r...' % (page,)
+                conn._HTTPConnection__state = _CS_IDLE # FU private variable!
+                conn.request("GET", page, None, headers)
+                respobjs[i] = conn.response_class(conn.sock, strict=conn.strict, method=conn._method)
+                out_bound += 1
+        # Try to read a response
+        for i,resp in enumerate(respobjs):
+            if resp is None:
+                continue
+            if debuglevel > 0:
+                print 'Retrieving %r...' % (pages[i],)
+            out_bound -= 1
+            skip_read = False
+            resp.begin()
+            if debuglevel > 0:
+                print '    %d %s' % (resp.status, resp.reason)
+            if 200 <= resp.status < 300:
+                # Ok
+                data[i] = resp.read()
+                cookie = resp.getheader('Set-Cookie')
+                if cookie is not None:
+                    headers['Cookie'] = cookie
+                skip_read = True
+                finished[i] = True
+                respobjs[i] = None
+            elif 300 <= resp.status < 400:
+                # Redirect
+                loc = resp.getheader('Location')
+                respobjs[i] = None
+                parsed = loc and urlparse.urlparse(loc)
+                if not parsed:
+                    # Missing or empty location header
+                    data[i] = (resp.status, resp.reason)
+                    finished[i] = True
+                elif parsed.netloc != '' and parsed.netloc != host:
+                    # Redirect to another host
+                    data[i] = (resp.status, resp.reason, loc)
+                    finished[i] = True
+                else:
+                    path = urlparse.urlunparse(parsed._replace(scheme='',netloc='',fragment=''))
+                    if debuglevel > 0:
+                        print '  Updated %r to %r' % (pages[i],path)
+                    pages[i] = path
+            elif resp.status >= 400:
+                # Failed
+                data[i] = (resp.status, resp.reason)
+                finished[i] = True
+                respobjs[i] = None
+            if resp.will_close:
+                # Connection (will be) closed, need to resend
+                conn.close()
+                if debuglevel > 0:
+                    print '  Connection closed'
+                for j,f in enumerate(finished):
+                    if not f and respobj[j] is not None:
+                        if debuglevel > 0:
+                            print '  Discarding out-bound request for %r' % (pages[j],)
+                        respobj[j] = None
+                break
+            elif not skip_read:
+                resp.read() # read any data
+            if any(not f and respobjs[j] is None for j,f in enumerate(finished)):
+                # Send another pending request
+                break
+        else:
+            break # All respobjs are None?
+    return data
+
+if __name__ == '__main__':
+    domain = 'en.wikipedia.org'
+    pages = ('/wiki/HTTP_pipelining', '/wiki/HTTP', '/wiki/HTTP_persistent_connection')
+    data = pipeline(domain,pages,max_out_bound=2,debuglevel=1)
+    for i,page in enumerate(data):
+        print
+        print '==== Page %r ====' % (pages[i],)
+        print page[:512]
+## end of http://code.activestate.com/recipes/576673/ }}}
+
diff --git a/toys/grbrute.py b/toys/brhute/grbrute.py
index fbdc8bc..fbdc8bc 100644
--- a/toys/grbrute.py
+++ b/toys/brhute/grbrute.py
diff --git a/toys/grbrute_profiling/profile_pphidden_async_data.txt b/toys/brhute/grbrute_profiling/profile_pphidden_async_data.txt
index cf85bda..cf85bda 100644
--- a/toys/grbrute_profiling/profile_pphidden_async_data.txt
+++ b/toys/brhute/grbrute_profiling/profile_pphidden_async_data.txt
diff --git a/toys/grbrute_profiling/profile_pphidden_async_data_sorted.txt b/toys/brhute/grbrute_profiling/profile_pphidden_async_data_sorted.txt
index 414c74d..414c74d 100644
--- a/toys/grbrute_profiling/profile_pphidden_async_data_sorted.txt
+++ b/toys/brhute/grbrute_profiling/profile_pphidden_async_data_sorted.txt
diff --git a/toys/grequests.py b/toys/brhute/grequests.py
index b10f0d4..b10f0d4 100644
--- a/toys/grequests.py
+++ b/toys/brhute/grequests.py
diff --git a/toys/httplib_pipelining.py b/toys/brhute/httplib_pipelining.py
index 5f5ac4e..5f5ac4e 100644
--- a/toys/httplib_pipelining.py
+++ b/toys/brhute/httplib_pipelining.py
diff --git a/toys/pphidden.py b/toys/brhute/pphidden.py
index cef18e2..cef18e2 100644
--- a/toys/pphidden.py
+++ b/toys/brhute/pphidden.py
diff --git a/toys/pphidden_async.py b/toys/brhute/pphidden_async.py
index 463364f..463364f 100644
--- a/toys/pphidden_async.py
+++ b/toys/brhute/pphidden_async.py
diff --git a/toys/threaded_resolver.py b/toys/brhute/threaded_resolver.py
index a2d347b..a2d347b 100644
--- a/toys/threaded_resolver.py
+++ b/toys/brhute/threaded_resolver.py
diff --git a/toys/twisted_http.py b/toys/brhute/twisted_http.py
index 805c78a..805c78a 100644
--- a/toys/twisted_http.py
+++ b/toys/brhute/twisted_http.py
diff --git a/toys/twisted_http_persistent.py b/toys/brhute/twisted_http_persistent.py
index 333bcb3..333bcb3 100644
--- a/toys/twisted_http_persistent.py
+++ b/toys/brhute/twisted_http_persistent.py
diff --git a/toys/twisted_http_simultaneous.py b/toys/brhute/twisted_http_simultaneous.py
index b9de196..b9de196 100644
--- a/toys/twisted_http_simultaneous.py
+++ b/toys/brhute/twisted_http_simultaneous.py