diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/app.py | 4 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 4 | ||||
| -rw-r--r-- | libmproxy/console/common.py | 5 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 7 | ||||
| -rw-r--r-- | libmproxy/flow.py | 41 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 9 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 10 | ||||
| -rw-r--r-- | libmproxy/protocol/primitives.py | 2 | ||||
| -rw-r--r-- | libmproxy/proxy.py | 602 | ||||
| -rw-r--r-- | libmproxy/proxy/__init__.py | 0 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 124 | ||||
| -rw-r--r-- | libmproxy/proxy/connection.py | 139 | ||||
| -rw-r--r-- | libmproxy/proxy/primitives.py | 40 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 287 |
14 files changed, 641 insertions, 633 deletions
diff --git a/libmproxy/app.py b/libmproxy/app.py index 24187704..69721031 100644 --- a/libmproxy/app.py +++ b/libmproxy/app.py @@ -17,12 +17,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-ca-cert.pem") + p = os.path.join(master().server.config.confdir, proxy.config.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-ca-cert.p12") + p = os.path.join(master().server.config.confdir, proxy.config.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 7950d40b..72c13769 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,4 +1,4 @@ -import proxy +from . import proxy import re, filt import argparse @@ -387,4 +387,4 @@ def common_options(parser): help="Allow access to users specified in an Apache htpasswd file." ) - proxy.ssl_option_group(parser) + proxy.config.ssl_option_group(parser) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 715bed80..7e2ecbf5 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -1,6 +1,7 @@ import urwid import urwid.util -from .. import utils, flow +from .. import utils +from ..protocol.http import CONTENT_MISSING VIEW_LIST = 0 @@ -183,7 +184,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): if f.response: if f.response.content: contentdesc = utils.pretty_size(len(f.response.content)) - elif f.response.content == flow.CONTENT_MISSING: + elif f.response.content == CONTENT_MISSING: contentdesc = "[content missing]" else: contentdesc = "[no content]" diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3486f57e..f5b5f83f 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -2,6 +2,7 @@ import os, sys, copy import urwid import common, grideditor, contentview from .. import utils, flow, controller +from ..protocol.http import CONTENT_MISSING class SearchError(Exception): pass @@ -150,7 +151,7 @@ class FlowView(common.WWrap): return (description, text_objects) def cont_view_handle_missing(self, conn, viewmode): - if conn.content == flow.CONTENT_MISSING: + if conn.content == CONTENT_MISSING: msg, body = "", [urwid.Text([("error", "[content missing]")])], 0 else: msg, body = self.content_view(viewmode, conn) @@ -178,7 +179,7 @@ class FlowView(common.WWrap): override = self.override_get() viewmode = self.viewmode_get(override) msg, body = self.cont_view_handle_missing(conn, viewmode) - elif conn.content == flow.CONTENT_MISSING: + elif conn.content == CONTENT_MISSING: pass return headers, msg, body @@ -643,7 +644,7 @@ class FlowView(common.WWrap): def delete_body(self, t): if t == "m": - val = flow.CONTENT_MISSING + val = CONTENT_MISSING else: val = None if self.state.view_flow_mode == common.VIEW_FLOW_REQUEST: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index f8ad2444..452fd783 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -7,14 +7,15 @@ import hashlib, Cookie, cookielib, re, threading import os import flask import requests +from . import controller, protocol +from .protocol import http +from .proxy.connection import ServerConnection +from .proxy.primitives import ProxyError import tnetstring, filt, script -from netlib import odict, wsgi -from .proxy import ClientConnection, ServerConnection # FIXME: remove circular dependency -import controller, version, protocol +from netlib import odict, wsgi, tcp +import netlib.http +import version import app -from .protocol import KILL -from .protocol.http import HTTPResponse, CONTENT_MISSING -from .proxy import RequestReplayThread ODict = odict.ODict ODictCaseless = odict.ODictCaseless @@ -565,7 +566,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = HTTPResponse._from_state(rflow.response._get_state()) + response = http.HTTPResponse._from_state(rflow.response._get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -642,7 +643,7 @@ class FlowMaster(controller.Master): """ if f.intercepting: return "Can't replay while intercepting..." - if f.request.content == CONTENT_MISSING: + if f.request.content == http.CONTENT_MISSING: return "Can't replay request with missing content..." if f.request: f.request.is_replay = True @@ -693,7 +694,7 @@ class FlowMaster(controller.Master): err = app.serve(r, r.flow.client_conn.wfile, **{"mitmproxy.master": self}) if err: self.add_event("Error in wsgi app. %s"%err, "error") - r.reply(KILL) + r.reply(protocol.KILL) return f = self.state.add_request(r) self.replacehooks.run(f) @@ -785,3 +786,25 @@ class FilteredFlowWriter: d = f._get_state() tnetstring.dump(d, self.fo) + +class RequestReplayThread(threading.Thread): + name="RequestReplayThread" + + def __init__(self, config, flow, masterq): + self.config, self.flow, self.channel = config, flow, controller.Channel(masterq) + threading.Thread.__init__(self) + + def run(self): + try: + r = self.flow.request + server = ServerConnection(self.flow.server_conn.address(), None) + server.connect() + if self.flow.server_conn.ssl_established: + server.establish_ssl(self.config.clientcerts, + self.flow.server_conn.sni) + server.send(r._assemble()) + self.flow.response = http.HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) + self.channel.ask("response", self.flow.response) + except (ProxyError, netlib.http.HttpError, tcp.NetLibError), v: + self.flow.error = protocol.primitives.Error(str(v)) + self.channel.ask("error", self.flow.error)
\ No newline at end of file diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index 2c2e7285..6200757f 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,14 +1,7 @@ -from ..proxy import ServerConnection, AddressPriority +from libmproxy.proxy.primitives import AddressPriority KILL = 0 # const for killed requests -class ConnectionTypeChange(Exception): - """ - Gets raised if the connetion type has been changed (e.g. after HTTP/1.1 101 Switching Protocols). - It's up to the raising ProtocolHandler to specify the new conntype before raising the exception. - """ - pass - class ProtocolHandler(object): def __init__(self, c): diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 8a2583b1..77a09e61 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,11 +1,13 @@ import Cookie, urllib, urlparse, time, copy from email.utils import parsedate_tz, formatdate, mktime_tz +from libmproxy.proxy.primitives import AddressPriority +from ..proxy.connection import ServerConnection +from ..proxy.primitives import ProxyError, ConnectionTypeChange import netlib.utils -from netlib import http, tcp, http_status, odict +from netlib import http, tcp, http_status from netlib.odict import ODict, ODictCaseless -from . import ProtocolHandler, ConnectionTypeChange, KILL, TemporaryServerChangeMixin -from .. import encoding, utils, version, filt, controller, stateobject -from ..proxy import ProxyError, AddressPriority, ServerConnection +from . import ProtocolHandler, KILL, TemporaryServerChangeMixin +from .. import encoding, utils, filt, controller, stateobject from .primitives import Flow, Error diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 90191eeb..f2701458 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -1,5 +1,5 @@ from .. import stateobject, utils, version -from ..proxy import ServerConnection, ClientConnection +from ..proxy.connection import ClientConnection, ServerConnection import copy diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py deleted file mode 100644 index 6dd37752..00000000 --- a/libmproxy/proxy.py +++ /dev/null @@ -1,602 +0,0 @@ -import os, socket, time, threading, copy -from OpenSSL import SSL -from netlib import tcp, http, certutils, http_auth -import utils, version, platform, controller, stateobject - -TRANSPARENT_SSL_PORTS = [443, 8443] -CONF_BASENAME = "mitmproxy" -CONF_DIR = "~/.mitmproxy" -CA_CERT_NAME = "mitmproxy-ca.pem" - - - -class AddressPriority(object): - """ - Enum that signifies the priority of the given address when choosing the destination host. - Higher is better (None < i) - """ - FORCE = 5 - """forward mode""" - MANUALLY_CHANGED = 4 - """user changed the target address in the ui""" - FROM_SETTINGS = 3 - """reverse proxy mode""" - FROM_CONNECTION = 2 - """derived from transparent resolver""" - FROM_PROTOCOL = 1 - """derived from protocol (e.g. absolute-form http requests)""" - - -class ProxyError(Exception): - def __init__(self, code, msg, headers=None): - self.code, self.msg, self.headers = code, msg, headers - - def __str__(self): - return "ProxyError(%s, %s)" % (self.code, self.msg) - - -class Log: - def __init__(self, msg): - self.msg = msg - - -class ProxyConfig: - def __init__(self, confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, - forward_proxy=None, transparent_proxy=None, authenticator=None, - ciphers=None, certs=None - ): - self.ciphers = ciphers - self.clientcerts = clientcerts - self.no_upstream_cert = no_upstream_cert - self.body_size_limit = body_size_limit - self.reverse_proxy = reverse_proxy - self.forward_proxy = forward_proxy - self.transparent_proxy = transparent_proxy - self.authenticator = authenticator - self.confdir = os.path.expanduser(confdir) - self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) - - - -class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): - def __init__(self, client_connection, address, server): - if client_connection: # Eventually, this object is restored from state. We don't have a connection then. - tcp.BaseHandler.__init__(self, client_connection, address, server) - else: - self.connection = None - self.server = None - self.wfile = None - self.rfile = None - self.address = None - self.clientcert = None - - self.timestamp_start = utils.timestamp() - self.timestamp_end = None - self.timestamp_ssl_setup = None - - _stateobject_attributes = dict( - timestamp_start=float, - timestamp_end=float, - timestamp_ssl_setup=float - ) - - def _get_state(self): - d = super(ClientConnection, self)._get_state() - d.update( - address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - clientcert=self.cert.to_pem() if self.clientcert else None - ) - return d - - def _load_state(self, state): - super(ClientConnection, self)._load_state(state) - self.address = tcp.Address(**state["address"]) if state["address"] else None - self.clientcert = certutils.SSLCert.from_pem(state["clientcert"]) if state["clientcert"] else None - - def copy(self): - return copy.copy(self) - - def send(self, message): - self.wfile.write(message) - self.wfile.flush() - - @classmethod - def _from_state(cls, state): - f = cls(None, tuple(), None) - f._load_state(state) - return f - - def convert_to_ssl(self, *args, **kwargs): - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - tcp.BaseHandler.finish(self) - self.timestamp_end = utils.timestamp() - - -class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): - def __init__(self, address, priority): - tcp.TCPClient.__init__(self, address) - self.priority = priority - - self.peername = None - self.timestamp_start = None - self.timestamp_end = None - self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None - - _stateobject_attributes = dict( - peername=tuple, - timestamp_start=float, - timestamp_end=float, - timestamp_tcp_setup=float, - timestamp_ssl_setup=float, - address=tcp.Address, - source_address=tcp.Address, - cert=certutils.SSLCert, - ssl_established=bool, - sni=str - ) - - def _get_state(self): - d = super(ServerConnection, self)._get_state() - d.update( - address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= {"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None, - cert=self.cert.to_pem() if self.cert else None - ) - return d - - def _load_state(self, state): - super(ServerConnection, self)._load_state(state) - - self.address = tcp.Address(**state["address"]) if state["address"] else None - self.source_address = tcp.Address(**state["source_address"]) if state["source_address"] else None - self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None - - @classmethod - def _from_state(cls, state): - f = cls(tuple(), None) - f._load_state(state) - return f - - def copy(self): - return copy.copy(self) - - def connect(self): - self.timestamp_start = utils.timestamp() - tcp.TCPClient.connect(self) - self.peername = self.connection.getpeername() - self.timestamp_tcp_setup = utils.timestamp() - - def send(self, message): - self.wfile.write(message) - self.wfile.flush() - - def establish_ssl(self, clientcerts, sni): - clientcert = None - if clientcerts: - path = os.path.join(clientcerts, self.address.host.encode("idna")) + ".pem" - if os.path.exists(path): - clientcert = path - try: - self.convert_to_ssl(cert=clientcert, sni=sni) - self.timestamp_ssl_setup = utils.timestamp() - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) - - def finish(self): - tcp.TCPClient.finish(self) - self.timestamp_end = utils.timestamp() - -from . import protocol -from .protocol.http import HTTPResponse - - -class RequestReplayThread(threading.Thread): - name="RequestReplayThread" - - def __init__(self, config, flow, masterq): - self.config, self.flow, self.channel = config, flow, controller.Channel(masterq) - threading.Thread.__init__(self) - - def run(self): - try: - r = self.flow.request - server = ServerConnection(self.flow.server_conn.address(), None) - server.connect() - if self.flow.server_conn.ssl_established: - server.establish_ssl(self.config.clientcerts, - self.flow.server_conn.sni) - server.send(r._assemble()) - self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) - self.channel.ask("response", self.flow.response) - except (ProxyError, http.HttpError, tcp.NetLibError), v: - self.flow.error = protocol.primitives.Error(str(v)) - self.channel.ask("error", self.flow.error) - - -class ConnectionHandler: - def __init__(self, config, client_connection, client_address, server, channel, server_version): - self.config = config - self.client_conn = ClientConnection(client_connection, client_address, server) - self.server_conn = None - self.channel, self.server_version = channel, server_version - - self.close = False - self.conntype = None - self.sni = None - - self.mode = "regular" - if self.config.reverse_proxy: - self.mode = "reverse" - if self.config.transparent_proxy: - self.mode = "transparent" - - def handle(self): - self.log("clientconnect") - self.channel.ask("clientconnect", self) - - self.determine_conntype() - - try: - try: - # Can we already identify the target server and connect to it? - server_address = None - address_priority = None - if self.config.forward_proxy: - server_address = self.config.forward_proxy[1:] - address_priority = AddressPriority.FORCE - elif self.config.reverse_proxy: - server_address = self.config.reverse_proxy[1:] - address_priority = AddressPriority.FROM_SETTINGS - elif self.config.transparent_proxy: - server_address = self.config.transparent_proxy["resolver"].original_addr( - self.client_conn.connection) - if not server_address: - raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") - address_priority = AddressPriority.FROM_CONNECTION - self.log("transparent to %s:%s" % server_address) - - if server_address: - self.set_server_address(server_address, address_priority) - self._handle_ssl() - - while not self.close: - try: - protocol.handle_messages(self.conntype, self) - except protocol.ConnectionTypeChange: - self.log("Connection Type Changed: %s" % self.conntype) - continue - - # FIXME: Do we want to persist errors? - except (ProxyError, tcp.NetLibError), e: - protocol.handle_error(self.conntype, self, e) - except Exception, e: - self.log(e.__class__) - import traceback - self.log(traceback.format_exc()) - self.log(str(e)) - - self.del_server_connection() - self.log("clientdisconnect") - self.channel.tell("clientdisconnect", self) - - def _handle_ssl(self): - """ - Helper function of .handle() - Check if we can already identify SSL connections. - If so, connect to the server and establish an SSL connection - """ - client_ssl = False - server_ssl = False - - if self.config.transparent_proxy: - client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) - elif self.config.reverse_proxy: - client_ssl = server_ssl = (self.config.reverse_proxy[0] == "https") - # TODO: Make protocol generic (as with transparent proxies) - # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) - if client_ssl or server_ssl: - self.establish_server_connection() - self.establish_ssl(client=client_ssl, server=server_ssl) - - def del_server_connection(self): - """ - Deletes an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.log("serverdisconnect", ["%s:%s" % (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - self.sni = None - - def determine_conntype(self): - #TODO: Add ruleset to select correct protocol depending on mode/target port etc. - self.conntype = "http" - - def set_server_address(self, address, priority): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - @type priority: AddressPriority - """ - address = tcp.Address.wrap(address) - - if self.server_conn: - if self.server_conn.priority > priority: - self.log("Attempt to change server address, " - "but priority is too low (is: %s, got: %s)" % (self.server_conn.priority, priority)) - return - if self.server_conn.address == address: - self.server_conn.priority = priority # Possibly increase priority - return - - self.del_server_connection() - - self.log("Set new server address: %s:%s" % (address.host, address.port)) - self.server_conn = ServerConnection(address, priority) - - def establish_server_connection(self): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - """ - if self.server_conn.connection: - return - self.log("serverconnect", ["%s:%s" % self.server_conn.address()[:2]]) - self.channel.tell("serverconnect", self) - try: - self.server_conn.connect() - except tcp.NetLibError, v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. If the target server is on the pass-through list, - the conntype attribute will be changed and the SSL connection won't be wrapped. - A protocol handler must raise a ConnTypeChanged exception if it detects that this is happening - """ - # TODO: Implement SSL pass-through handling and change conntype - passthrough = [ - # "echo.websocket.org", - # "174.129.224.73" # echo.websocket.org, transparent mode - ] - if self.server_conn.address.host in passthrough or self.sni in passthrough: - self.conntype = "tcp" - return - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % self.sni) - self.log("Establish SSL", subs) - - if server: - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") - self.establish_server_connection() # make sure there is a server connection. - self.server_conn.establish_ssl(self.config.clientcerts, self.sni) - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key = self.find_cert() - self.client_conn.convert_to_ssl( - cert, key, - handle_sni = self.handle_sni, - cipher_list = self.config.ciphers - ) - - def server_reconnect(self, no_ssl=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - priority = self.server_conn.priority - sni = self.sni - self.log("(server reconnect follows)") - self.del_server_connection() - self.set_server_address(address, priority) - self.establish_server_connection() - if had_ssl and not no_ssl: - self.sni = sni - self.establish_ssl(server=True) - - def finish(self): - self.client_conn.finish() - - def log(self, msg, subs=()): - msg = [ - "%s:%s: %s" % (self.client_conn.address.host, self.client_conn.address.port, msg) - ] - for i in subs: - msg.append(" -> " + i) - msg = "\n".join(msg) - self.channel.tell("log", Log(msg)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if not self.config.no_upstream_cert or not self.server_conn.ssl_established: - upstream_cert = self.server_conn.cert - if upstream_cert.cn: - host = upstream_cert.cn.decode("utf8").encode("idna") - sans = upstream_cert.altnames - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if sn and sn != self.sni: - self.sni = sn.decode("utf8").encode("idna") - self.log("SNI received: %s" % self.sni) - self.server_reconnect() # reconnect to upstream server with SNI - # Now, change client context to reflect changed certificate: - new_context = SSL.Context(SSL.TLSv1_METHOD) - cert, key = self.find_cert() - new_context.use_privatekey_file(key) - new_context.use_certificate(cert.X509) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except Exception, e: # pragma: no cover - pass - - -class ProxyServerError(Exception): - pass - - -class ProxyServer(tcp.TCPServer): - allow_reuse_address = True - bound = True - def __init__(self, config, port, host='', server_version=version.NAMEVERSION): - """ - Raises ProxyServerError if there's a startup problem. - """ - self.config = config - self.server_version = server_version - try: - tcp.TCPServer.__init__(self, (host, port)) - except socket.error, v: - raise ProxyServerError('Error starting proxy server: ' + v.strerror) - self.channel = None - - def start_slave(self, klass, channel): - slave = klass(channel, self) - slave.start() - - def set_channel(self, channel): - self.channel = channel - - def handle_client_connection(self, conn, client_address): - h = ConnectionHandler(self.config, conn, client_address, self, self.channel, self.server_version) - h.handle() - h.finish() - - -class DummyServer: - bound = False - - def __init__(self, config): - self.config = config - - def start_slave(self, *args): - pass - - def shutdown(self): - pass - - -# Command-line utils -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", dest='certs', default=[], type=str, - metavar = "SPEC", action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". '\ - 'The domain may include a wildcard, and is equal to "*" if not specified. '\ - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, '\ - 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--ciphers", action="store", - type=str, dest="ciphers", default=None, - help="SSL cipher specification." - ) - - -def process_proxy_options(parser, options): - body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - return parser.error("Can't set both reverse proxy and transparent proxy.") - - if options.transparent_proxy: - if not platform.resolver: - return parser.error("Transparent mode not supported on this platform.") - trans = dict( - resolver=platform.resolver(), - sslports=TRANSPARENT_SSL_PORTS - ) - else: - trans = None - - if options.reverse_proxy: - rp = utils.parse_proxy_spec(options.reverse_proxy) - if not rp: - return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy) - else: - rp = None - - if options.forward_proxy: - fp = utils.parse_proxy_spec(options.forward_proxy) - if not fp: - return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy) - else: - fp = None - - if options.clientcerts: - options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): - return parser.error( - "Client certificate directory does not exist or is not a directory: %s" % options.clientcerts - ) - - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): - if options.auth_singleuser: - if len(options.auth_singleuser.split(':')) != 2: - return parser.error("Invalid single-user specification. Please use the format username:password") - username, password = options.auth_singleuser.split(':') - password_manager = http_auth.PassManSingleUser(username, password) - elif options.auth_nonanonymous: - password_manager = http_auth.PassManNonAnon() - elif options.auth_htpasswd: - try: - password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) - except ValueError, v: - return parser.error(v.message) - authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") - else: - authenticator = http_auth.NullProxyAuth(None) - - certs = [] - for i in options.certs: - parts = i.split("=", 1) - if len(parts) == 1: - parts = ["*", parts[0]] - parts[1] = os.path.expanduser(parts[1]) - if not os.path.exists(parts[1]): - parser.error("Certificate file does not exist: %s"%parts[1]) - certs.append(parts) - - return ProxyConfig( - clientcerts=options.clientcerts, - body_size_limit=body_size_limit, - no_upstream_cert=options.no_upstream_cert, - reverse_proxy=rp, - forward_proxy=fp, - transparent_proxy=trans, - authenticator=authenticator, - ciphers=options.ciphers, - certs = certs, - ) diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/libmproxy/proxy/__init__.py diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py new file mode 100644 index 00000000..38c6ce89 --- /dev/null +++ b/libmproxy/proxy/config.py @@ -0,0 +1,124 @@ +import os +from .. import utils, platform +from netlib import http_auth, certutils + + +TRANSPARENT_SSL_PORTS = [443, 8443] +CONF_BASENAME = "mitmproxy" +CONF_DIR = "~/.mitmproxy" + + +class ProxyConfig: + def __init__(self, confdir=CONF_DIR, clientcerts=None, + no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, + forward_proxy=None, transparent_proxy=None, authenticator=None, + ciphers=None, certs=None + ): + self.ciphers = ciphers + self.clientcerts = clientcerts + self.no_upstream_cert = no_upstream_cert + self.body_size_limit = body_size_limit + self.reverse_proxy = reverse_proxy + self.forward_proxy = forward_proxy + self.transparent_proxy = transparent_proxy + self.authenticator = authenticator + self.confdir = os.path.expanduser(confdir) + self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) + + +def process_proxy_options(parser, options): + body_size_limit = utils.parse_size(options.body_size_limit) + if options.reverse_proxy and options.transparent_proxy: + return parser.error("Can't set both reverse proxy and transparent proxy.") + + if options.transparent_proxy: + if not platform.resolver: + return parser.error("Transparent mode not supported on this platform.") + trans = dict( + resolver=platform.resolver(), + sslports=TRANSPARENT_SSL_PORTS + ) + else: + trans = None + + if options.reverse_proxy: + rp = utils.parse_proxy_spec(options.reverse_proxy) + if not rp: + return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy) + else: + rp = None + + if options.forward_proxy: + fp = utils.parse_proxy_spec(options.forward_proxy) + if not fp: + return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy) + else: + fp = None + + if options.clientcerts: + options.clientcerts = os.path.expanduser(options.clientcerts) + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): + return parser.error( + "Client certificate directory does not exist or is not a directory: %s" % options.clientcerts + ) + + if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_singleuser: + if len(options.auth_singleuser.split(':')) != 2: + return parser.error("Invalid single-user specification. Please use the format username:password") + username, password = options.auth_singleuser.split(':') + password_manager = http_auth.PassManSingleUser(username, password) + elif options.auth_nonanonymous: + password_manager = http_auth.PassManNonAnon() + elif options.auth_htpasswd: + try: + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + except ValueError, v: + return parser.error(v.message) + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + else: + authenticator = http_auth.NullProxyAuth(None) + + certs = [] + for i in options.certs: + parts = i.split("=", 1) + if len(parts) == 1: + parts = ["*", parts[0]] + parts[1] = os.path.expanduser(parts[1]) + if not os.path.exists(parts[1]): + parser.error("Certificate file does not exist: %s"%parts[1]) + certs.append(parts) + + return ProxyConfig( + clientcerts=options.clientcerts, + body_size_limit=body_size_limit, + no_upstream_cert=options.no_upstream_cert, + reverse_proxy=rp, + forward_proxy=fp, + transparent_proxy=trans, + authenticator=authenticator, + ciphers=options.ciphers, + certs = certs, + ) + + +def ssl_option_group(parser): + group = parser.add_argument_group("SSL") + group.add_argument( + "--cert", dest='certs', default=[], type=str, + metavar = "SPEC", action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". '\ + 'The domain may include a wildcard, and is equal to "*" if not specified. '\ + 'The file at path is a certificate in PEM format. If a private key is included in the PEM, '\ + 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' + ) + group.add_argument( + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." + ) + group.add_argument( + "--ciphers", action="store", + type=str, dest="ciphers", default=None, + help="SSL cipher specification." + )
\ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py new file mode 100644 index 00000000..3a0273af --- /dev/null +++ b/libmproxy/proxy/connection.py @@ -0,0 +1,139 @@ +import copy +import os +from .. import stateobject, utils +from .primitives import ProxyError +from netlib import tcp, certutils + + +class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): + def __init__(self, client_connection, address, server): + if client_connection: # Eventually, this object is restored from state. We don't have a connection then. + tcp.BaseHandler.__init__(self, client_connection, address, server) + else: + self.connection = None + self.server = None + self.wfile = None + self.rfile = None + self.address = None + self.clientcert = None + + self.timestamp_start = utils.timestamp() + self.timestamp_end = None + self.timestamp_ssl_setup = None + + _stateobject_attributes = dict( + timestamp_start=float, + timestamp_end=float, + timestamp_ssl_setup=float + ) + + def _get_state(self): + d = super(ClientConnection, self)._get_state() + d.update( + address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, + clientcert=self.cert.to_pem() if self.clientcert else None + ) + return d + + def _load_state(self, state): + super(ClientConnection, self)._load_state(state) + self.address = tcp.Address(**state["address"]) if state["address"] else None + self.clientcert = certutils.SSLCert.from_pem(state["clientcert"]) if state["clientcert"] else None + + def copy(self): + return copy.copy(self) + + def send(self, message): + self.wfile.write(message) + self.wfile.flush() + + @classmethod + def _from_state(cls, state): + f = cls(None, tuple(), None) + f._load_state(state) + return f + + def convert_to_ssl(self, *args, **kwargs): + tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + tcp.BaseHandler.finish(self) + self.timestamp_end = utils.timestamp() + + +class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): + def __init__(self, address, priority): + tcp.TCPClient.__init__(self, address) + self.priority = priority + + self.peername = None + self.timestamp_start = None + self.timestamp_end = None + self.timestamp_tcp_setup = None + self.timestamp_ssl_setup = None + + _stateobject_attributes = dict( + peername=tuple, + timestamp_start=float, + timestamp_end=float, + timestamp_tcp_setup=float, + timestamp_ssl_setup=float, + address=tcp.Address, + source_address=tcp.Address, + cert=certutils.SSLCert, + ssl_established=bool, + sni=str + ) + + def _get_state(self): + d = super(ServerConnection, self)._get_state() + d.update( + address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, + source_address= {"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None, + cert=self.cert.to_pem() if self.cert else None + ) + return d + + def _load_state(self, state): + super(ServerConnection, self)._load_state(state) + + self.address = tcp.Address(**state["address"]) if state["address"] else None + self.source_address = tcp.Address(**state["source_address"]) if state["source_address"] else None + self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None + + @classmethod + def _from_state(cls, state): + f = cls(tuple(), None) + f._load_state(state) + return f + + def copy(self): + return copy.copy(self) + + def connect(self): + self.timestamp_start = utils.timestamp() + tcp.TCPClient.connect(self) + self.peername = self.connection.getpeername() + self.timestamp_tcp_setup = utils.timestamp() + + def send(self, message): + self.wfile.write(message) + self.wfile.flush() + + def establish_ssl(self, clientcerts, sni): + clientcert = None + if clientcerts: + path = os.path.join(clientcerts, self.address.host.encode("idna")) + ".pem" + if os.path.exists(path): + clientcert = path + try: + self.convert_to_ssl(cert=clientcert, sni=sni) + self.timestamp_ssl_setup = utils.timestamp() + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) + + def finish(self): + tcp.TCPClient.finish(self) + self.timestamp_end = utils.timestamp()
\ No newline at end of file diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py new file mode 100644 index 00000000..8dd0e16a --- /dev/null +++ b/libmproxy/proxy/primitives.py @@ -0,0 +1,40 @@ +class ProxyError(Exception): + def __init__(self, code, msg, headers=None): + self.code, self.msg, self.headers = code, msg, headers + + def __str__(self): + return "ProxyError(%s, %s)" % (self.code, self.msg) + + +class ConnectionTypeChange(Exception): + """ + Gets raised if the connection type has been changed (e.g. after HTTP/1.1 101 Switching Protocols). + It's up to the raising ProtocolHandler to specify the new conntype before raising the exception. + """ + pass + + +class ProxyServerError(Exception): + pass + + +class AddressPriority(object): + """ + Enum that signifies the priority of the given address when choosing the destination host. + Higher is better (None < i) + """ + FORCE = 5 + """forward mode""" + MANUALLY_CHANGED = 4 + """user changed the target address in the ui""" + FROM_SETTINGS = 3 + """reverse proxy mode""" + FROM_CONNECTION = 2 + """derived from transparent resolver""" + FROM_PROTOCOL = 1 + """derived from protocol (e.g. absolute-form http requests)""" + + +class Log: + def __init__(self, msg): + self.msg = msg
\ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py new file mode 100644 index 00000000..37ec7758 --- /dev/null +++ b/libmproxy/proxy/server.py @@ -0,0 +1,287 @@ +import socket +from .. import version, protocol +from libmproxy.proxy.primitives import Log +from .primitives import ProxyServerError +from .connection import ClientConnection, ServerConnection +from .primitives import ProxyError, ConnectionTypeChange, AddressPriority +from netlib import tcp + + +class DummyServer: + bound = False + + def __init__(self, config): + self.config = config + + def start_slave(self, *args): + pass + + def shutdown(self): + pass + + +class ProxyServer(tcp.TCPServer): + allow_reuse_address = True + bound = True + def __init__(self, config, port, host='', server_version=version.NAMEVERSION): + """ + Raises ProxyServerError if there's a startup problem. + """ + self.config = config + self.server_version = server_version + try: + tcp.TCPServer.__init__(self, (host, port)) + except socket.error, v: + raise ProxyServerError('Error starting proxy server: ' + v.strerror) + self.channel = None + + def start_slave(self, klass, channel): + slave = klass(channel, self) + slave.start() + + def set_channel(self, channel): + self.channel = channel + + def handle_client_connection(self, conn, client_address): + h = ConnectionHandler(self.config, conn, client_address, self, self.channel, self.server_version) + h.handle() + h.finish() + + +class ConnectionHandler: + def __init__(self, config, client_connection, client_address, server, channel, server_version): + self.config = config + self.client_conn = ClientConnection(client_connection, client_address, server) + self.server_conn = None + self.channel, self.server_version = channel, server_version + + self.close = False + self.conntype = None + self.sni = None + + self.mode = "regular" + if self.config.reverse_proxy: + self.mode = "reverse" + if self.config.transparent_proxy: + self.mode = "transparent" + + def handle(self): + self.log("clientconnect") + self.channel.ask("clientconnect", self) + + self.determine_conntype() + + try: + try: + # Can we already identify the target server and connect to it? + server_address = None + address_priority = None + if self.config.forward_proxy: + server_address = self.config.forward_proxy[1:] + address_priority = AddressPriority.FORCE + elif self.config.reverse_proxy: + server_address = self.config.reverse_proxy[1:] + address_priority = AddressPriority.FROM_SETTINGS + elif self.config.transparent_proxy: + server_address = self.config.transparent_proxy["resolver"].original_addr( + self.client_conn.connection) + if not server_address: + raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + address_priority = AddressPriority.FROM_CONNECTION + self.log("transparent to %s:%s" % server_address) + + if server_address: + self.set_server_address(server_address, address_priority) + self._handle_ssl() + + while not self.close: + try: + protocol.handle_messages(self.conntype, self) + except ConnectionTypeChange: + self.log("Connection Type Changed: %s" % self.conntype) + continue + + # FIXME: Do we want to persist errors? + except (ProxyError, tcp.NetLibError), e: + protocol.handle_error(self.conntype, self, e) + except Exception, e: + self.log(e.__class__) + import traceback + self.log(traceback.format_exc()) + self.log(str(e)) + + self.del_server_connection() + self.log("clientdisconnect") + self.channel.tell("clientdisconnect", self) + + def _handle_ssl(self): + """ + Helper function of .handle() + Check if we can already identify SSL connections. + If so, connect to the server and establish an SSL connection + """ + client_ssl = False + server_ssl = False + + if self.config.transparent_proxy: + client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) + elif self.config.reverse_proxy: + client_ssl = server_ssl = (self.config.reverse_proxy[0] == "https") + # TODO: Make protocol generic (as with transparent proxies) + # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) + if client_ssl or server_ssl: + self.establish_server_connection() + self.establish_ssl(client=client_ssl, server=server_ssl) + + def del_server_connection(self): + """ + Deletes an existing server connection. + """ + if self.server_conn and self.server_conn.connection: + self.server_conn.finish() + self.log("serverdisconnect", ["%s:%s" % (self.server_conn.address.host, self.server_conn.address.port)]) + self.channel.tell("serverdisconnect", self) + self.server_conn = None + self.sni = None + + def determine_conntype(self): + #TODO: Add ruleset to select correct protocol depending on mode/target port etc. + self.conntype = "http" + + def set_server_address(self, address, priority): + """ + Sets a new server address with the given priority. + Does not re-establish either connection or SSL handshake. + @type priority: libmproxy.proxy.primitives.AddressPriority + """ + address = tcp.Address.wrap(address) + + if self.server_conn: + if self.server_conn.priority > priority: + self.log("Attempt to change server address, " + "but priority is too low (is: %s, got: %s)" % (self.server_conn.priority, priority)) + return + if self.server_conn.address == address: + self.server_conn.priority = priority # Possibly increase priority + return + + self.del_server_connection() + + self.log("Set new server address: %s:%s" % (address.host, address.port)) + self.server_conn = ServerConnection(address, priority) + + def establish_server_connection(self): + """ + Establishes a new server connection. + If there is already an existing server connection, the function returns immediately. + """ + if self.server_conn.connection: + return + self.log("serverconnect", ["%s:%s" % self.server_conn.address()[:2]]) + self.channel.tell("serverconnect", self) + try: + self.server_conn.connect() + except tcp.NetLibError, v: + raise ProxyError(502, v) + + def establish_ssl(self, client=False, server=False): + """ + Establishes SSL on the existing connection(s) to the server or the client, + as specified by the parameters. If the target server is on the pass-through list, + the conntype attribute will be changed and the SSL connection won't be wrapped. + A protocol handler must raise a ConnTypeChanged exception if it detects that this is happening + """ + # TODO: Implement SSL pass-through handling and change conntype + passthrough = [ + # "echo.websocket.org", + # "174.129.224.73" # echo.websocket.org, transparent mode + ] + if self.server_conn.address.host in passthrough or self.sni in passthrough: + self.conntype = "tcp" + return + + # Logging + if client or server: + subs = [] + if client: + subs.append("with client") + if server: + subs.append("with server (sni: %s)" % self.sni) + self.log("Establish SSL", subs) + + if server: + if self.server_conn.ssl_established: + raise ProxyError(502, "SSL to Server already established.") + self.establish_server_connection() # make sure there is a server connection. + self.server_conn.establish_ssl(self.config.clientcerts, self.sni) + if client: + if self.client_conn.ssl_established: + raise ProxyError(502, "SSL to Client already established.") + cert, key = self.find_cert() + self.client_conn.convert_to_ssl( + cert, key, + handle_sni = self.handle_sni, + cipher_list = self.config.ciphers + ) + + def server_reconnect(self, no_ssl=False): + address = self.server_conn.address + had_ssl = self.server_conn.ssl_established + priority = self.server_conn.priority + sni = self.sni + self.log("(server reconnect follows)") + self.del_server_connection() + self.set_server_address(address, priority) + self.establish_server_connection() + if had_ssl and not no_ssl: + self.sni = sni + self.establish_ssl(server=True) + + def finish(self): + self.client_conn.finish() + + def log(self, msg, subs=()): + msg = [ + "%s:%s: %s" % (self.client_conn.address.host, self.client_conn.address.port, msg) + ] + for i in subs: + msg.append(" -> " + i) + msg = "\n".join(msg) + self.channel.tell("log", Log(msg)) + + def find_cert(self): + host = self.server_conn.address.host + sans = [] + if not self.config.no_upstream_cert or not self.server_conn.ssl_established: + upstream_cert = self.server_conn.cert + if upstream_cert.cn: + host = upstream_cert.cn.decode("utf8").encode("idna") + sans = upstream_cert.altnames + + ret = self.config.certstore.get_cert(host, sans) + if not ret: + raise ProxyError(502, "Unable to generate dummy cert.") + return ret + + def handle_sni(self, connection): + """ + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). We now connect upstream to + figure out which certificate needs to be served. + """ + try: + sn = connection.get_servername() + if sn and sn != self.sni: + self.sni = sn.decode("utf8").encode("idna") + self.log("SNI received: %s" % self.sni) + self.server_reconnect() # reconnect to upstream server with SNI + # Now, change client context to reflect changed certificate: + new_context = SSL.Context(SSL.TLSv1_METHOD) + cert, key = self.find_cert() + new_context.use_privatekey_file(key) + new_context.use_certificate(cert.X509) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except Exception, e: # pragma: no cover + pass
\ No newline at end of file |