privoxy-tls/proxy.py

389 lines
14 KiB
Python
Raw Normal View History

2015-01-07 04:57:51 +01:00
#!/usr/bin/env python3
2019-05-07 12:06:21 +02:00
"HTTP Proxy utilities, pyOpenSSL version"
2015-01-07 04:57:51 +01:00
__author__ = 'phoenix'
__version__ = '1.0'
from datetime import datetime
import logging
2015-12-11 21:49:48 +01:00
import threading
2015-01-07 04:57:51 +01:00
import cgi
import socket
import select
import ssl
2019-05-07 12:06:21 +02:00
2015-01-07 04:57:51 +01:00
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn
2019-05-07 12:06:21 +02:00
from cert import get_cert
2015-01-07 04:57:51 +01:00
2019-05-07 12:06:21 +02:00
_name = 'proxy'
2015-01-07 04:57:51 +01:00
logger = logging.getLogger('__main__')
2019-05-07 12:06:21 +02:00
message_format = '''\
<!doctype html>
2015-01-07 04:57:51 +01:00
<html>
2019-05-07 12:06:21 +02:00
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8">
<title>Proxy Error: {code}</title>
</head>
<body>
<h1>{code}: {message}</h1>
<p>The following error occurred while
trying to access <strong>{url}</strong>
</p>
<p><strong>{explain}</strong></p>
<hr>Generated on {now} by {server}.
</body>
2015-01-07 04:57:51 +01:00
</html>
2019-05-07 12:06:21 +02:00
'''
2015-01-07 04:57:51 +01:00
def read_write(socket1, socket2, max_idling=10):
2019-05-07 12:06:21 +02:00
'''
Read and Write contents between 2 sockets
'''
2015-01-07 04:57:51 +01:00
iw = [socket1, socket2]
ow = []
count = 0
while True:
count += 1
(ins, _, exs) = select.select(iw, ow, iw, 1)
2019-05-07 12:06:21 +02:00
if exs:
break
2015-01-07 04:57:51 +01:00
if ins:
for reader in ins:
writer = socket2 if reader is socket1 else socket1
try:
data = reader.recv(1024)
if data:
writer.send(data)
count = 0
2019-05-07 12:06:21 +02:00
except (ConnectionAbortedError,
ConnectionResetError,
BrokenPipeError):
2015-01-07 04:57:51 +01:00
pass
2019-05-07 12:06:21 +02:00
if count == max_idling:
break
2015-01-07 04:57:51 +01:00
2015-12-11 21:49:48 +01:00
class Counter:
reset_value = 999
2019-05-07 12:06:21 +02:00
2015-12-11 21:49:48 +01:00
def __init__(self, start=0):
self.lock = threading.Lock()
self.value = start
2019-05-07 12:06:21 +02:00
2015-12-11 21:49:48 +01:00
def increment_and_set(self, obj, attr):
with self.lock:
self.value = self.value + 1 if self.value < self.reset_value else 1
setattr(obj, attr, self.value)
2015-01-07 04:57:51 +01:00
class ProxyRequestHandler(BaseHTTPRequestHandler):
2019-05-07 12:06:21 +02:00
'''
RequestHandler with do_CONNECT method defined
'''
server_version = f'{_name}/{__version__}'
2015-01-07 04:57:51 +01:00
# do_CONNECT() will set self.ssltunnel to override this
ssltunnel = False
# Override default value 'HTTP/1.0'
protocol_version = 'HTTP/1.1'
2015-12-11 21:49:48 +01:00
# To be set in each request
reqNum = 0
2015-01-07 04:57:51 +01:00
def do_CONNECT(self):
2019-05-07 12:06:21 +02:00
'''
Descrypt https request and dispatch to http handler
'''
2015-01-07 04:57:51 +01:00
# request line: CONNECT www.example.com:443 HTTP/1.1
2019-05-07 12:06:21 +02:00
self.host, self.port = self.path.split(':')
2019-05-14 19:47:54 +02:00
# TLS MITM
2019-05-07 12:06:21 +02:00
self.wfile.write(
('HTTP/1.1 200 Connection established\r\n'
+ f'Proxy-agent: {self.version_string()}\r\n'
+ '\r\n').encode('ascii'))
if self.host.count('.') >= 2:
commonname = '.' + self.host.partition('.')[-1]
else:
commonname = self.host
dummycert = get_cert(commonname, self.config)
2015-01-07 04:57:51 +01:00
# set a flag for do_METHOD
self.ssltunnel = True
2019-05-07 12:06:21 +02:00
ssl_sock = ssl.wrap_socket(self.connection, keyfile=dummycert,
certfile=dummycert, server_side=True)
2015-01-07 04:57:51 +01:00
# Ref: Lib/socketserver.py#StreamRequestHandler.setup()
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.rbufsize)
self.wfile = self.connection.makefile('wb', self.wbufsize)
# dispatch to do_METHOD()
self.handle_one_request()
def handle_one_request(self):
2019-05-07 12:06:21 +02:00
'''Catch more exceptions than default
2015-01-07 04:57:51 +01:00
Intend to catch exceptions on local side
Exceptions on remote side should be handled in do_*()
2019-05-07 12:06:21 +02:00
'''
2015-01-07 04:57:51 +01:00
try:
BaseHTTPRequestHandler.handle_one_request(self)
return
except (ConnectionError, FileNotFoundError) as e:
2019-05-07 12:06:21 +02:00
logger.warning('{:03d} {} {}'.format(
self.reqNum, self.server_version, e))
2015-01-07 04:57:51 +01:00
except (ssl.SSLEOFError, ssl.SSLError) as e:
if hasattr(self, 'url'):
# Happens after the tunnel is established
2019-05-07 12:06:21 +02:00
logger.warning(f'{self.reqNum:03d} "{e}" while operating'
2019-05-14 19:47:54 +02:00
f' on established local TSL tunnel for'
2019-05-07 12:06:21 +02:00
f' [{self.url}]')
2015-01-07 04:57:51 +01:00
else:
2019-05-07 12:06:21 +02:00
logger.warning(f'{self.reqNum:03d} "{e}" while trying to'
2019-05-14 19:47:54 +02:00
' establish local TLS tunnel for'
2019-05-07 12:06:21 +02:00
f'[{self.path}]')
2015-01-07 04:57:51 +01:00
self.close_connection = 1
def sendout_error(self, url, code, message=None, explain=None):
2019-05-07 12:06:21 +02:00
'''
Modified from http.server.send_error() for customized display
'''
2015-01-07 04:57:51 +01:00
try:
shortmsg, longmsg = self.responses[code]
except KeyError:
shortmsg, longmsg = '???', '???'
if message is None:
message = shortmsg
if explain is None:
explain = longmsg
2019-05-07 12:06:21 +02:00
content = message_format.format(
code=code, message=message,
explain=explain, url=url,
now=datetime.today(),
server=self.server_version)
2015-01-07 04:57:51 +01:00
body = content.encode('UTF-8', 'replace')
self.send_response_only(code, message)
2019-05-07 12:06:21 +02:00
self.send_header('Content-Type', self.error_content_type)
2015-01-07 04:57:51 +01:00
self.send_header('Content-Length', int(len(body)))
self.end_headers()
if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
self.wfile.write(body)
def deny_request(self):
self.send_response_only(403)
self.send_header('Content-Length', 0)
self.end_headers()
def redirect(self, url):
self.send_response_only(302)
self.send_header('Content-Length', 0)
self.send_header('Location', url)
self.end_headers()
def forward_to_https_proxy(self):
2019-05-07 12:06:21 +02:00
'''
Forward https request to upstream https proxy
'''
logger.debug(f'Using Proxy - {self.proxy}')
2015-01-07 04:57:51 +01:00
proxy_host, proxy_port = self.proxy.split('//')[1].split(':')
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
server_conn.connect((proxy_host, int(proxy_port)))
2019-05-07 12:06:21 +02:00
server_conn.send((
f'CONNECT {self.path} HTTP/1.1\r\n\r\n').encode('ascii'))
2015-01-07 04:57:51 +01:00
server_conn.settimeout(0.1)
datas = b''
while True:
try:
data = server_conn.recv(4096)
except socket.timeout:
break
if data:
datas += data
else:
break
server_conn.setblocking(True)
if b'200' in datas and b'established' in datas.lower():
2019-05-14 19:47:54 +02:00
logger.info('{:03d} [P] TLS passthru: https://{}/'.format(
2019-05-07 12:06:21 +02:00
self.reqNum, self.path))
self.wfile.write(('HTTP/1.1 200 Connection established\r\n' +
'Proxy-agent: {}\r\n\r\n'.format(
self.version_string()).encode('ascii')))
2015-01-07 04:57:51 +01:00
read_write(self.connection, server_conn)
else:
2019-05-07 12:06:21 +02:00
logger.warning('{:03d} Proxy {} failed.'.format(
self.reqNum, self.proxy))
2015-01-07 04:57:51 +01:00
if datas:
logger.debug(datas)
self.wfile.write(datas)
finally:
2019-05-07 12:06:21 +02:00
# We don't maintain a connection reuse pool,
# so close the connection anyway
2015-01-07 04:57:51 +01:00
server_conn.close()
def forward_to_socks5_proxy(self):
2019-05-07 12:06:21 +02:00
'''
Forward https request to upstream socks5 proxy
'''
logger.warning('Socks5 proxy not implemented yet, '
'please use https proxy')
2015-01-07 04:57:51 +01:00
def tunnel_traffic(self):
2019-05-07 12:06:21 +02:00
'''
Tunnel traffic to remote host:port
'''
2019-05-14 19:47:54 +02:00
logger.info('{:03d} [D] TLS passthru: https://{}/'.format(
2019-05-07 12:06:21 +02:00
self.reqNum, self.path))
2015-01-07 04:57:51 +01:00
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
server_conn.connect((self.host, int(self.port)))
2019-05-07 12:06:21 +02:00
self.wfile.write(('HTTP/1.1 200 Connection established\r\n'
+ 'Proxy-agent: {}\r\n'.format(
self.version_string())
+ '\r\n').encode('ascii'))
2015-01-07 04:57:51 +01:00
read_write(self.connection, server_conn)
except TimeoutError:
2019-05-07 12:06:21 +02:00
self.wfile.write(b'HTTP/1.1 504 Gateway Timeout\r\n\r\n')
logger.warning('{:03d} Timed Out: https://{}:{}/'.format(
self.reqNum, self.host, self.port))
2015-01-07 04:57:51 +01:00
except socket.gaierror as e:
2019-05-07 12:06:21 +02:00
self.wfile.write(b'HTTP/1.1 503 Service Unavailable\r\n\r\n')
logger.warning('{:03d} {}: https://{}:{}/'.format(
self.reqNum, e, self.host, self.port))
2015-01-07 04:57:51 +01:00
finally:
2019-05-07 12:06:21 +02:00
# We don't maintain a connection reuse pool,
# so close the connection anyway
2015-01-07 04:57:51 +01:00
server_conn.close()
def ssl_get_response(self, conn):
try:
2019-05-07 12:06:21 +02:00
server_conn = ssl.wrap_socket(
conn, cert_reqs=ssl.CERT_REQUIRED,
ca_certs="cacert.pem",
ssl_version=ssl.PROTOCOL_TLSv1)
server_conn.sendall(
('{} {} HTTP/1.1\r\n'.format(
self.command, self.path)).encode('ascii'))
2015-01-07 04:57:51 +01:00
server_conn.sendall(self.headers.as_bytes())
if self.postdata:
server_conn.sendall(self.postdata)
while True:
data = server_conn.recv(4096)
if data:
self.wfile.write(data)
2019-05-07 12:06:21 +02:00
else:
break
2015-01-07 04:57:51 +01:00
except (ssl.SSLEOFError, ssl.SSLError) as e:
2019-05-07 12:06:21 +02:00
logger.error("[SSLError]")
self.send_error(417, message="Exception "
+ str(e.__class__), explain=str(e))
2015-01-07 04:57:51 +01:00
def purge_headers(self, headers):
2019-05-07 12:06:21 +02:00
'''
Remove hop-by-hop headers that shouldn't pass through a Proxy
'''
for name in ['Connection', 'Keep-Alive', 'Upgrade',
'Proxy-Connection', 'Proxy-Authenticate']:
2015-01-07 04:57:51 +01:00
del headers[name]
2015-12-11 21:49:48 +01:00
def purge_write_headers(self, headers):
2015-01-07 04:57:51 +01:00
self.purge_headers(headers)
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
2019-05-07 12:06:21 +02:00
2015-01-07 04:57:51 +01:00
def stream_to_client(self, response):
bufsize = 1024 * 64
need_chunked = 'Transfer-Encoding' in response.headers
written = 0
while True:
data = response.read(bufsize)
if not data:
if need_chunked:
self.wfile.write(b'0\r\n\r\n')
break
if need_chunked:
2019-05-07 12:06:21 +02:00
self.wfile.write((f'{len(data):x}\r\n').encode('ascii'))
2015-01-07 04:57:51 +01:00
self.wfile.write(data)
if need_chunked:
self.wfile.write(b'\r\n')
written += len(data)
return written
2019-05-07 12:06:21 +02:00
2015-01-07 04:57:51 +01:00
def http_request_info(self):
2019-05-07 12:06:21 +02:00
'''
Return HTTP request information in bytes
'''
context = ['CLIENT VALUES:',
f'client_address = {self.client_address}',
f'requestline = {self.requestline}',
f'command = {self.command}',
f'path = {self.path}',
f'request_version = {self.request_version}',
'',
'SERVER VALUES:',
'server_version = {self.server_version}',
'sys_version = {self.sys_version}',
'protocol_version = {self.protocol_version}',
'',
'HEADER RECEIVED:']
2015-01-07 04:57:51 +01:00
for name, value in sorted(self.headers.items()):
2019-05-07 12:06:21 +02:00
context.append(f'{name} = {value.rstrip()}')
2015-01-07 04:57:51 +01:00
2019-05-07 12:06:21 +02:00
if self.command == 'POST':
context.append('\r\nPOST VALUES:')
2015-01-07 04:57:51 +01:00
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD': 'POST'})
for field in form.keys():
fielditem = form[field]
if fielditem.filename:
# The field contains an uploaded file
file_data = fielditem.file.read()
file_len = len(file_data)
2019-05-07 12:06:21 +02:00
context.append('Uploaded {} as "{}" ({} bytes)'.format(
field, fielditem.filename, file_len))
2015-01-07 04:57:51 +01:00
else:
# Regular form value
2019-05-07 12:06:21 +02:00
context.append(f'{field} = {fielditem.value}')
return('\r\n'.join(context).encode('ascii'))
2015-01-07 04:57:51 +01:00
def demo():
PORT = 8000
class ProxyServer(ThreadingMixIn, HTTPServer):
2019-05-07 12:06:21 +02:00
'''Handle requests in a separate thread.'''
2015-01-07 04:57:51 +01:00
pass
class RequestHandler(ProxyRequestHandler):
2019-05-07 12:06:21 +02:00
'Displaying HTTP request information'
server_version = 'DemoProxy/0.1'
2015-01-07 04:57:51 +01:00
def do_METHOD(self):
2019-05-07 12:06:21 +02:00
'Universal method for GET, POST, HEAD, PUT and DELETE'
2015-01-07 04:57:51 +01:00
message = self.http_request_info()
self.send_response(200)
# 'Content-Length' is important for HTTP/1.1
self.send_header('Content-Length', len(message))
self.end_headers()
self.wfile.write(message)
2019-05-07 12:06:21 +02:00
do_GET = do_POST = do_HEAD = do_METHOD
do_PUT = do_DELETE = do_OPTIONS = do_METHOD
2015-01-07 04:57:51 +01:00
2019-05-07 12:06:21 +02:00
print(RequestHandler.server_version, 'serving now, <Ctrl-C> to stop ...')
print(f'Listen Addr : localhost:{PORT}')
print('-' * 10)
2015-01-07 04:57:51 +01:00
server = ProxyServer(('', PORT), RequestHandler)
server.serve_forever()
2019-05-07 12:06:21 +02:00
2015-01-07 04:57:51 +01:00
if __name__ == '__main__':
try:
demo()
except KeyboardInterrupt:
2019-05-07 12:06:21 +02:00
print('Quitting...')