privoxy-tls/main.py

#!/usr/bin/env python3

"A Privoxy Helper Program"

__author__ = 'phoenix'
__version__ = 'v1.4'


from socketserver import ThreadingMixIn
from http.server import HTTPServer
from urllib.parse import urlparse
from proxy import Counter, ProxyRequestHandler, get_cert
from urllib3.contrib.socks import SOCKSProxyManager

import os
import time
import configparser
import fnmatch
import logging
import threading
import ssl
import urllib3
import argparse
import collections


_name = 'privoxy-tls'


class LoadConfig:
    def __init__(self, configfile):
        self.config = configparser.ConfigParser(
            allow_no_value=True, delimiters=('=',),
            inline_comment_prefixes=('#',))
        self.config.read(configfile)
        self.proxy_name = self.config['general'].get('proxAddr')
        self.front_name = self.config['general'].get('frontAddr', 'localhost')
        self.rear_name  = self.config['general'].get('rearAddr', 'localhost')
        self.front_port = int(self.config['general'].get('frontPort'))
        self.rear_port  = int(self.config['general'].get('rearPort'))
        self.proxy      = self.config['general'].get('defaultProxy')
        self.loglevel   = self.config['general'].get('logLevel')
        self.ca         = self.config['general'].get('caCert')
        self.certdir    = self.config['general'].get('certDir')


class ConnectionPools:
    """
    self.pools is a list of {'proxy': 'http://127.0.0.1:8080',
                             'pool': urllib3.ProxyManager() object,
                             'patterns': ['ab.com', 'bc.net', ...]}
    self.getpool() is a method that returns pool based on host matching
    """
    sslparams = dict(
        cert_reqs="REQUIRED",
        ca_certs='/etc/ssl/certs/ca-bundle.crt')
    timeout = urllib3.util.timeout.Timeout(connect=90.0, read=90.0)

    def __init__(self, config):
        self.file = config
        self.file_timestamp = os.path.getmtime(config)
        self.loadConfig()

    def loadConfig(self):
        # self.conf has to be inited each time for reloading
        self.conf = configparser.ConfigParser(
            allow_no_value=True, delimiters=('=',),
            inline_comment_prefixes=('#',))
        self.conf.read(self.file)
        self.pools = []
        proxy_sections = [section for section in self.conf.sections()
                          if section.startswith('proxy')]
        for section in proxy_sections:
            proxy = section.split()[1]
            self.pools.append(dict(proxy=proxy,
                                   pool=self.setProxyPool(proxy),
                                   patterns=list(self.conf[section].keys())))
        default_proxy = self.conf['general'].get('defaultProxy')

        if default_proxy:
            default_pool = self.setProxyPool(default_proxy)
        else:
            default_pool = [
                urllib3.PoolManager(num_pools=10, maxsize=8,
                                    timeout=self.timeout, **self.sslparams),
                urllib3.PoolManager(num_pools=10, maxsize=8,
                                    timeout=self.timeout)]

        self.pools.append(dict(proxy=default_proxy,
                               pool=default_pool, patterns='*'))

        # handle missing sections
        sections = collections.defaultdict(dict)
        for name in self.conf.sections():
            sections[name] = self.conf[name]
        self.noverifylist = list(sections['noVerify'].keys())
        self.sslpasslist  = list(sections['passthru'].keys())
        self.blacklist    = list(sections['blacklist'].keys())
        self.bypasslist   = list(sections['bypassURL'].keys())

    def reloadConfig(self):
        while True:
            mtime = os.path.getmtime(self.file)
            if mtime > self.file_timestamp:
                self.file_timestamp = mtime
                self.loadConfig()
                logger.info("*" * 20 + " CONFIG RELOADED " + "*" * 20)
            time.sleep(1)

    def getpool(self, host, httpmode=False):
        things = (fnmatch.fnmatch(host, pattern)
                  for pattern in self.noverifylist)
        noverify = True if httpmode or any(things) else False
        logger.debug(f'host: {host}, noverify: {noverify}')
        for pool in self.pools:
            things = (fnmatch.fnmatch(host, pattern)
                      for pattern in pool['patterns'])
            if any(things):
                return pool['proxy'], pool['pool'][noverify], noverify

    def setProxyPool(self, proxy):
        scheme = proxy.split(':')[0]
        if scheme in ('http', 'https'):
            ProxyManager = urllib3.ProxyManager
        elif scheme in ('socks4', 'socks5'):
            ProxyManager = SOCKSProxyManager
        else:
            print("Wrong Proxy Format: " + proxy)
            print("Proxy should start with http/https/socks4/socks5 .")
            input()
            raise SystemExit
        # maxsize is the max. number of connections to the same server
        return [
            ProxyManager(proxy, num_pools=10, maxsize=8, timeout=self.timeout,
                         **self.sslparams),
            ProxyManager(proxy, num_pools=10, maxsize=8, timeout=self.timeout)]


class FrontServer(ThreadingMixIn, HTTPServer):
    """Handle requests in a separate thread."""
    pass


class RearServer(ThreadingMixIn, HTTPServer):
    """Handle requests in a separate thread."""
    pass


class FrontRequestHandler(ProxyRequestHandler):
    """
    Sit between the client and Privoxy
    Convert https request to http
    """
    server_version = f'{_name} front/{__version__}'

    def do_CONNECT(self):
        "Descrypt https request and dispatch to http handler"

        # request line: CONNECT www.example.com:443 HTTP/1.1
        self.host, self.port = self.path.split(":")
        self.proxy, self.pool, self.noverify = pools.getpool(self.host)
        things = (fnmatch.fnmatch(self.host, pattern)
                  for pattern in pools.blacklist)
        if any(things):
            # blacklist
            self.deny_request()
            logger.info('{:03d} denied by blacklist: {}'.format(
                        self.reqNum, self.host))
        elif any((fnmatch.fnmatch(self.host, pattern)
                 for pattern in pools.sslpasslist)):
            # TLS passthru
            if self.proxy and self.proxy.startswith('https'):
                self.forward_to_https_proxy()
            elif self.proxy and self.proxy.startswith('socks5'):
                self.forward_to_socks5_proxy()
            else:
                self.tunnel_traffic()
            # Upstream server or proxy of the tunnel is
            # closed explictly, so we close the local connection too
            self.close_connection = 1
        else:
            # TLS MITM
            self.wfile.write(('HTTP/1.1 200 Connection established\r\n'
                              f'Proxy-agent: {self.version_string()}\r\n'
                              '\r\n').encode('ascii'))
            if self.host.count('.') >= 2:
                isccSLD = len(self.host.split('.')[-2]) <= 3
                if isccSLD:
                    commonname = self.host
                else:
                    commonname = '.' + self.host.partition('.')[-1]
            else:
                commonname = self.host
            dummycert = get_cert(commonname, config)
            # set a flag for do_METHOD
            self.ssltunnel = True

            ssl_sock = ssl.wrap_socket(self.connection, keyfile=dummycert,
                                       certfile=dummycert, server_side=True)
            # Ref: Lib/socketserver.py#StreamRequestHandler.setup()
            self.connection = ssl_sock
            self.rfile = self.connection.makefile('rb', self.rbufsize)
            self.wfile = self.connection.makefile('wb', self.wbufsize)
            # dispatch to do_METHOD()
            self.handle_one_request()

    def do_METHOD(self):
        "Forward request to Privoxy"

        counter.increment_and_set(self, 'reqNum')

        if self.ssltunnel:
            # https request
            host = (self.host if self.port == '443'
                    else ':'.join((self.host, self.port)))
            url = 'https://' + ''.join((host, self.path))
            self.bypass = any((fnmatch.fnmatch(url, pattern)
                              for pattern in pools.bypasslist))
            if not self.bypass:
                url = 'http://' + ''.join((host, self.path))
                # Tag the request so Privoxy can recognize it
                self.headers["Tagged"] = (self.version_string()
                                          + f':{self.reqNum:d}')
        else:
            # http request
            self.host = urlparse(self.path).hostname
            if any((fnmatch.fnmatch(self.host, pattern)
                   for pattern in pools.blacklist)):
                # blacklist
                self.deny_request()
                logger.info('{:03d} denied by blacklist: {}'.format(
                            self.reqNum, self.host))
                return
            host = urlparse(self.path).netloc
            self.proxy, self.pool, self.noverify = pools.getpool(
                self.host, httpmode=True)
            self.bypass = any((fnmatch.fnmatch('http://' + host +
                               urlparse(self.path).path, pattern)
                              for pattern in pools.bypasslist))
            url = self.path
        self.url = url
        pool = self.pool if self.bypass else proxpool
        data_length = self.headers.get("Content-Length")
        self.postdata = (self.rfile.read(int(data_length))
                         if data_length and int(data_length) > 0 else None)
        if self.command == "POST" and "Content-Length" not in self.headers:
            buffer = self.rfile.read()
            if buffer:
                logger.warning(
                    '{:03d} '
                    'POST w/o "Content-Length" header (Bytes: {} | '
                    'Transfer-Encoding: {} | '
                    'HTTPS: {}'.format(
                        self.reqNum, len(buffer),
                        "Transfer-Encoding" in self.headers,
                        self.ssltunnel))
        # Remove hop-by-hop headers
        self.purge_headers(self.headers)
        r = None

        # Merge the proxy headers. Only do this in HTTP. We have to copy the
        # headers dict so we can safely change it without those changes being
        # reflected in anyone else's copy.
        # if self.scheme == 'http':
        #     headers = headers.copy()
        #     headers.update(self.proxy_headers)
        headers = urllib3._collections.HTTPHeaderDict(self.headers)

        try:
            # Sometimes 302 redirect would fail with "BadStatusLine"
            # exception, and IE11 doesn't restart the request.
            # retries=1 instead of retries=False fixes it.
            # ! Retry may cause the requests with the same reqNum appear
            # in the log window
            r = pool.urlopen(
                self.command, url, body=self.postdata, headers=headers,
                retries=1, redirect=False, preload_content=False,
                decode_content=False)
            if not self.ssltunnel:
                if self.bypass:
                    prefix = '[BP]' if self.proxy else '[BD]'
                else:
                    prefix = '[D]'
                if self.command in ("GET", "HEAD"):
                    logger.info('{:03d} {} "{} {}" {} {}'.format(
                                self.reqNum, prefix, self.command, url,
                                r.status, r.getheader('Content-Length', '-')))
                else:
                    logger.info('{:03d} {} "{} {} {}" {} {}'.format(
                                self.reqNum, prefix, self.command, url,
                                data_length, r.status,
                                r.getheader('Content-Length', '-')))

            self.send_response_only(r.status, r.reason)
            # HTTPResponse.msg is easier to handle
            # than urllib3._collections.HTTPHeaderDict
            r.headers = r._original_response.msg
            self.purge_write_headers(r.headers)

            if (self.command == 'HEAD' or r.status in (100, 101, 204, 304)
               or r.getheader("Content-Length") == '0'):
                written = None
            else:
                written = self.stream_to_client(r)
                if ("Content-Length" not in r.headers
                   and 'Transfer-Encoding' not in r.headers):
                    self.close_connection = 1

        # Intend to catch regular http and bypass
        # http/https requests exceptions
        # Regular https request exceptions should be handled by rear server
        except urllib3.exceptions.TimeoutError as e:
            self.sendout_error(url, 504, message="Timeout", explain=e)
            logger.warning(f'{self.reqNum:03d} [F] {e} on '
                           f'"{self.command} {url}"')
        except (urllib3.exceptions.HTTPError,) as e:
            self.sendout_error(url, 502, message="HTTP Error", explain=e)
            logger.warning(f'{self.reqNum:03d} [F] {e} on '
                           f'"{self.command} {url}"')
        finally:
            if r:
                # Release the connection back into the pool
                r.release_conn()

    do_GET = do_POST = do_HEAD = do_PUT = do_DELETE = do_OPTIONS = do_METHOD


class RearRequestHandler(ProxyRequestHandler):
    """
    Supposed to be the parent proxy for Privoxy for tagged requests
    Convert http request to https
    """
    server_version = f'{_name} rear/{__version__}'

    def do_METHOD(self):
        "Convert http request to https"

        if (self.headers.get("Tagged")
           and self.headers["Tagged"].startswith(_name)):
            self.reqNum = int(self.headers["Tagged"].split(":")[1])
            # Remove the tag
            del self.headers["Tagged"]
        else:
            self.sendout_error(
                self.path, 400,
                explain='The proxy setting of the client'
                ' is misconfigured.\n\n'
                f'Please set the HTTPS proxy port to {config.FRONTPORT} '
                'and check the Docs for other settings.')
            logger.error("[Misconfigured HTTPS proxy port] " + self.path)
            return

        # request line: GET http://somehost.com/path?attr=value HTTP/1.1
        url = "https" + self.path[4:]
        self.host = urlparse(self.path).hostname
        proxy, pool, noverify = pools.getpool(self.host)
        prefix = '[P]' if proxy else '[D]'
        data_length = self.headers.get("Content-Length")
        self.postdata = (self.rfile.read(int(data_length))
                         if data_length else None)
        self.purge_headers(self.headers)
        r = None

        # Below code in connectionpool.py expect the headers
        # to has a copy() and update() method
        # That's why we can't use self.headers directly when
        # call pool.urlopen()
        #
        # Merge the proxy headers. Only do this in HTTP. We have to copy the
        # headers dict so we can safely change it without those changes being
        # reflected in anyone else's copy.
        # if self.scheme == 'http':
        #     headers = headers.copy()
        #     headers.update(self.proxy_headers)
        headers = urllib3._collections.HTTPHeaderDict(self.headers)

        try:
            r = pool.urlopen(
                self.command, url, body=self.postdata, headers=headers,
                retries=1, redirect=False, preload_content=False,
                decode_content=False)
            if proxy:
                logger.debug('Using Proxy - ' + proxy)
            if self.command in ("GET", "HEAD"):
                logger.info('{:03d} {} "{} {}" {} {}'.format(
                            self.reqNum, prefix,
                            self.command, url, r.status,
                            r.getheader('Content-Length', '-')))
            else:
                logger.info('{:03d} {} "{} {} {}" {} {}'.format(
                            self.reqNum, prefix,
                            self.command, url, data_length, r.status,
                            r.getheader('Content-Length', '-')))

            self.send_response_only(r.status, r.reason)
            # HTTPResponse.msg is easier to handle than
            # urllib3._collections.HTTPHeaderDict
            r.headers = r._original_response.msg
            self.purge_write_headers(r.headers)

            if (self.command == 'HEAD' or r.status in (100, 101, 204, 304)
               or r.getheader("Content-Length") == '0'):
                written = None
            else:
                written = self.stream_to_client(r)
                if ("Content-Length" not in r.headers
                   and 'Transfer-Encoding' not in r.headers):
                    self.close_connection = 1

        except urllib3.exceptions.SSLError as e:
            self.sendout_error(url, 417, message='TLS Certificate Failed',
                               explain=e)
            logger.error(f'{self.reqNum:03d} [TLS Certificate Error] {url}')
        except urllib3.exceptions.TimeoutError as e:
            self.sendout_error(url, 504, message='Timeout', explain=e)
            logger.warning(f'{self.reqNum:03d} [R]{prefix} '
                           f'"{self.command} {url}" {e}')
        except (urllib3.exceptions.HTTPError,) as e:
            self.sendout_error(url, 502, message="HTTP Error", explain=e)
            logger.warning(f'{self.reqNum:03d} [R]{prefix} '
                           f'"{self.command} {url}" {e}')

        finally:
            if r:
                # Release the connection back into the pool
                r.release_conn()

    do_GET = do_POST = do_HEAD = do_PUT = do_DELETE = do_OPTIONS = do_METHOD


def main():
    urllib3.disable_warnings()
    logger.setLevel(getattr(logging, config.loglevel, logging.INFO))
    handler = logging.StreamHandler()
    formatter = logging.Formatter('%(asctime)s %(message)s', datefmt='[%H:%M]')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    frontserver = FrontServer((config.front_name, config.front_port),
                              FrontRequestHandler)
    rearserver = RearServer((config.rear_name, config.rear_port),
                            RearRequestHandler)
    for worker in (frontserver.serve_forever,
                   rearserver.serve_forever,
                   pools.reloadConfig):
        thread = threading.Thread(target=worker)
        thread.daemon = True
        thread.start()

    print('=' * 40)
    print(f'{_name} {__version__} (urllib3/{urllib3.__version__})')
    print(f'Front   : {config.front_name}:{config.front_port}')
    print(f'Privoxy : {config.proxy_name}')
    print(f'Rear    : {config.rear_name}:{config.rear_port}')
    print(f'Proxy   : {config.proxy}')
    print('=' * 40)
    while True:
        time.sleep(1)


if __name__ == '__main__':
    parser = argparse.ArgumentParser('Privoxy TLS proxy wrapper.')
    parser.add_argument('-c', '--config', type=argparse.FileType('r'),
                        default='config.ini',
                        help='Privoxy TLS configuration file.')
    args = parser.parse_args()

    # globals
    CONFIG = args.config.name
    logger = logging.getLogger(__name__)
    config = LoadConfig(CONFIG)
    proxpool = urllib3.ProxyManager(
        config.proxy_name, num_pools=10, maxsize=8,
        timeout=urllib3.util.timeout.Timeout(
            connect=90.0, read=310.0))
    pools = ConnectionPools(CONFIG)
    counter = Counter()
    try:
        main()
    except KeyboardInterrupt:
        print("Quitting...")