instrumentation/next-share/BaseLib/Core/BitTornado/zurllib.py

   1 # Written by John Hoffman
   2 # see LICENSE.txt for license information
   3
   4 import sys
   5 from httplib import HTTPConnection, HTTPSConnection, HTTPException
   6 from urlparse import urlparse
   7 from bencode import bdecode
   8 from gzip import GzipFile
   9 from StringIO import StringIO
  10 from __init__ import product_name, version_short
  11 from traceback import print_exc,print_stack
  12
  13 from BaseLib.Core.Utilities.timeouturlopen import find_proxy
  14
  15 VERSION = product_name+'/'+version_short
  16 MAX_REDIRECTS = 10
  17
  18
  19 class btHTTPcon(HTTPConnection): # attempt to add automatic connection timeout
  20     def connect(self):
  21         HTTPConnection.connect(self)
  22         try:
  23             self.sock.settimeout(30)
  24         except:
  25             pass
  26
  27 class btHTTPScon(HTTPSConnection): # attempt to add automatic connection timeout
  28     def connect(self):
  29         HTTPSConnection.connect(self)
  30         try:
  31             self.sock.settimeout(30)
  32         except:
  33             pass
  34
  35 class urlopen:
  36     def __init__(self, url):
  37         self.tries = 0
  38         self._open(url.strip())
  39         self.error_return = None
  40
  41     def _open(self, url):
  42         try:
  43             self.tries += 1
  44             if self.tries > MAX_REDIRECTS:
  45                 raise IOError, ('http error', 500,
  46                                 "Internal Server Error: Redirect Recursion")
  47             (scheme, netloc, path, pars, query, fragment) = urlparse(url)
  48             if scheme != 'http' and scheme != 'https':
  49                 raise IOError, ('url error', 'unknown url type', scheme, url)
  50             wanturl = path
  51             if pars:
  52                 wanturl += ';'+pars
  53             if query:
  54                 wanturl += '?'+query
  55     #        if fragment:
  56
  57             proxyhost = find_proxy(url)
  58             if proxyhost is None:
  59                 desthost = netloc
  60                 desturl = wanturl
  61             else:
  62                 desthost = proxyhost
  63                 desturl = scheme+'://'+netloc+wanturl
  64             try:
  65                 self.response = None
  66                 if scheme == 'http':
  67                     self.connection = btHTTPcon(desthost)
  68                 else:
  69                     self.connection = btHTTPScon(desthost)
  70                 self.connection.request('GET', desturl, None,
  71                                     { 'Host': netloc, 'User-Agent': VERSION,
  72                                       'Accept-Encoding': 'gzip' } )
  73                 self.response = self.connection.getresponse()
  74             except HTTPException, e:
  75                 print_exc()
  76                 raise IOError, ('http error', str(e))
  77             status = self.response.status
  78             if status in (301, 302):
  79                 try:
  80                     self.connection.close()
  81                 except:
  82                     pass
  83                 self._open(self.response.getheader('Location'))
  84                 return
  85             if status != 200:
  86                 try:
  87                     data = self._read()
  88                     d = bdecode(data)
  89                     if d.has_key('failure reason'):
  90                         self.error_return = data
  91                         return
  92                 except:
  93                     pass
  94                 raise IOError, ('http error', status, self.response.reason)
  95         except:
  96             print_exc()
  97
  98
  99     def read(self):
 100         if self.error_return:
 101             return self.error_return
 102         return self._read()
 103
 104     def _read(self):
 105         data = self.response.read()
 106         if self.response.getheader('Content-Encoding', '').find('gzip') >= 0:
 107             try:
 108                 compressed = StringIO(data)
 109                 f = GzipFile(fileobj = compressed)
 110                 data = f.read()
 111             except:
 112                 raise IOError, ('http error', 'got corrupt response')
 113         return data
 114
 115     def close(self):
 116         self.connection.close()
 117
 118 try:
 119     import pycurl
 120
 121     class curlopen:
 122             def __init__(self,url):
 123
 124                     print >>sys.stderr,"CURL",url
 125
 126                     self.contents = ''
 127                     self.c = pycurl.Curl()
 128                     self.c.setopt(c.URL, url)
 129                     self.c.setopt(c.WRITEFUNCTION, t.body_callback)
 130                     self.c.perform()
 131                     self.c.close()
 132
 133             def body_callback(self, buf):
 134                     self.contents = self.contents + buf
 135
 136             def read(self):
 137                 return self.contents
 138
 139             def close(self):
 140                 pass
 141 except:
 142     pass
 143