1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92 | # urllib2 opener to connection through a proxy using the CONNECT method, (useful for SSL)
# tested with python 2.4
import urllib2
import urllib
import httplib
import socket
class ProxyHTTPConnection(httplib.HTTPConnection):
_ports = {'http' : 80, 'https' : 443}
def request(self, method, url, body=None, headers={}):
#request is called before connect, so can interpret url and get
#real host/port to be used to make CONNECT request to proxy
proto, rest = urllib.splittype(url)
if proto is None:
raise ValueError, "unknown URL type: %s" % url
#get host
host, rest = urllib.splithost(rest)
#try to get port
host, port = urllib.splitport(host)
#if port is not defined try to get from proto
if port is None:
try:
port = self._ports[proto]
except KeyError:
raise ValueError, "unknown protocol for: %s" % url
self._real_host = host
self._real_port = port
httplib.HTTPConnection.request(self, method, url, body, headers)
def connect(self):
httplib.HTTPConnection.connect(self)
#send proxy CONNECT request
self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self._real_host, self._real_port))
#expect a HTTP/1.0 200 Connection established
response = self.response_class(self.sock, strict=self.strict, method=self._method)
(version, code, message) = response._read_status()
#probably here we can handle auth requests...
if code != 200:
#proxy returned and error, abort connection, and raise exception
self.close()
raise socket.error, "Proxy connection failed: %d %s" % (code, message.strip())
#eat up header block from proxy....
while True:
#should not use directly fp probablu
line = response.fp.readline()
if line == '\r\n': break
class ProxyHTTPSConnection(ProxyHTTPConnection):
default_port = 443
def __init__(self, host, port = None, key_file = None, cert_file = None, strict = None):
ProxyHTTPConnection.__init__(self, host, port)
self.key_file = key_file
self.cert_file = cert_file
def connect(self):
ProxyHTTPConnection.connect(self)
#make the sock ssl-aware
ssl = socket.ssl(self.sock, self.key_file, self.cert_file)
self.sock = httplib.FakeSocket(self.sock, ssl)
class ConnectHTTPHandler(urllib2.HTTPHandler):
def do_open(self, http_class, req):
return urllib2.HTTPHandler.do_open(self, ProxyHTTPConnection, req)
class ConnectHTTPSHandler(urllib2.HTTPSHandler):
def do_open(self, http_class, req):
return urllib2.HTTPSHandler.do_open(self, ProxyHTTPSConnection, req)
if __name__ == '__main__':
import sys
opener = urllib2.build_opener(ConnectHTTPHandler, ConnectHTTPSHandler)
urllib2.install_opener(opener)
req = urllib2.Request(url='https://192.168.1.1')
req.set_proxy('192.168.1.254:3128', 'https')
f = urllib2.urlopen(req)
print f.read()
|
Download
Copy to clipboard
Modification to use with https redirection (required for libgmail).
HTTP proxy with CONNECT support to tunnel HTTPS request. To work with "HTTP proxy with CONNECT support to tunnel HTTPS request", a ProxyHTTPSConnertion is needed inside ConnectHTTPHandler
This is the code change
urllib2.HTTPSHandler error. i'm getting this error:
I'm using Python 2.4
i am behind a proxy which requires authentication how to use opener for that
To do proxy authentication I edited connect as follows:
No matter what I tried, I couldn't get the auth handlers to do the right thing. I can't wait until I need to add SOCKS support....
I have posted a working Python 3 version here. It was generated using the 2to3 syntax tool then all the bits that didn't work (ssl, http.client, utf-8 vs ascii issues) were fixed. It also shows how to add handlers to the chain for the proxy and basic authentication.
Cliffs notes version: Send the connect string (and read the response) in utf-8 format: self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self._real_host, self._real_port)) becomes: connect_string="CONNECT {0}:{1} HTTP/1.0\r\n\r\n".format(self._real_host, self._real_port) self.send(connect_string.encode('utf-8'))
Wrap the socket for ssl using the ssl class (instead of a method in socket class): ssl = socket.ssl(self.sock, self.key_file, self.cert_file) self.sock = httplib.FakeSocket(self.sock, ssl) becomes: self.sock = ssl.wrap_socket(self.sock, self.key_file, self.cert_file)
I have tested this for get and post.
Hello,
For better compatibility with servers, the Request URI should not contain the location (scheme, fqdn, port, etc...) but only the path, therefore the following code:
must be replaced by:
Regards, Rm4dillo
Take care with this loop:
It is not uncommon that your proxy might return an empty response, which would cause this loop to never terminate.
You can use httpsproxy_urllib2 (available on pypi) for python2.4 or python2.5 instead this recipe. For me it works better.
@boussard
Works great! Thanks for sharing!