diff options
Diffstat (limited to 'libmproxy')
29 files changed, 2035 insertions, 925 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 6125bfbf..7f6f69ef 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address, sslversion_choices -from netlib import http import netlib.utils from . import filt, utils, version @@ -108,26 +108,9 @@ def parse_server_spec(url): raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) - - -def parse_server_spec_special(url): - """ - Provides additional support for http2https and https2http schemes. - """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def get_common_options(options): @@ -192,24 +175,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +284,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +358,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ - ) + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) - - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ + group.add_argument( + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) + +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +455,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +464,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +530,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +548,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +573,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +597,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +625,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -633,20 +675,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -663,13 +705,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/console/statusbar.py b/libmproxy/console/statusbar.py index 7eb2131b..ea2dbfa8 100644 --- a/libmproxy/console/statusbar.py +++ b/libmproxy/console/statusbar.py @@ -199,11 +199,12 @@ class StatusBar(urwid.WidgetWrap): r.append("[%s]" % (":".join(opts))) if self.master.server.config.mode in ["reverse", "upstream"]: - dst = self.master.server.config.mode.dst - scheme = "https" if dst[0] else "http" - if dst[1] != dst[0]: - scheme += "2https" if dst[1] else "http" - r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:])) + dst = self.master.server.config.upstream_server + r.append("[dest:%s]" % netlib.utils.unparse_url( + dst.scheme, + dst.address.host, + dst.address.port + )) if self.master.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index 3b0f7512..e5ce11da 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -8,3 +8,7 @@ jsbeautifier, git checkout 25/03/12, MIT license wbxml - https://github.com/davidpshaw/PyWBXMLDecoder + +tls, BSD license + - https://github.com/mhils/tls/tree/mitmproxy + - limited to required files.
\ No newline at end of file diff --git a/libmproxy/contrib/tls/__init__.py b/libmproxy/contrib/tls/__init__.py new file mode 100644 index 00000000..4b540884 --- /dev/null +++ b/libmproxy/contrib/tls/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py new file mode 100644 index 00000000..9c57a799 --- /dev/null +++ b/libmproxy/contrib/tls/_constructs.py @@ -0,0 +1,213 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from construct import (Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, TunnelAdapter, GreedyRange, + Switch, OptionalGreedyRange) + +from .utils import UBInt24 + +ProtocolVersion = Struct( + "version", + UBInt8("major"), + UBInt8("minor"), +) + +TLSPlaintext = Struct( + "TLSPlaintext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCompressed = Struct( + "TLSCompressed", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 1024 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCiphertext = Struct( + "TLSCiphertext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 2048 + Bytes("fragment", lambda ctx: ctx.length), +) + +Random = Struct( + "random", + UBInt32("gmt_unix_time"), + Bytes("random_bytes", 28), +) + +SessionID = Struct( + "session_id", + UBInt8("length"), + Bytes("session_id", lambda ctx: ctx.length), +) + +CipherSuites = Struct( + "cipher_suites", + UBInt16("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, UBInt16("cipher_suites")), +) + +CompressionMethods = Struct( + "compression_methods", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("compression_methods")), +) + +ServerName = Struct( + "", + UBInt8("type"), + PascalString("name", length_field=UBInt16("length")), +) + +SNIExtension = Struct( + "", + TunnelAdapter( + PascalString("server_names", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(ServerName) + ), + ), +) + +ALPNExtension = Struct( + "", + TunnelAdapter( + PascalString("alpn_protocols", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(PascalString("name")) + ), + ), +) + +UnknownExtension = Struct( + "", + PascalString("bytes", length_field=UBInt16("extensions_length")) +) + +Extension = Struct( + "Extension", + UBInt16("type"), + Embed( + Switch( + "", lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension + }, + default=UnknownExtension + ) + ) +) + +extensions = TunnelAdapter( + PascalString("extensions", length_field=UBInt16("extensions_length")), + OptionalGreedyRange(Extension) +) + +ClientHello = Struct( + "ClientHello", + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = Struct( + "ServerHello", + ProtocolVersion, + Random, + SessionID, + Bytes("cipher_suite", 2), + UBInt8("compression_method"), + extensions, +) + +ClientCertificateType = Struct( + "certificate_types", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("certificate_types")), +) + +SignatureAndHashAlgorithm = Struct( + "algorithms", + UBInt8("hash"), + UBInt8("signature"), +) + +SupportedSignatureAlgorithms = Struct( + "supported_signature_algorithms", + UBInt16("supported_signature_algorithms_length"), + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = Struct( + "certificate_authorities", + UBInt16("length"), + Bytes("certificate_authorities", lambda ctx: ctx.length), +) + +CertificateRequest = Struct( + "CertificateRequest", + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = Struct( + "ServerDHParams", + UBInt16("dh_p_length"), + Bytes("dh_p", lambda ctx: ctx.dh_p_length), + UBInt16("dh_g_length"), + Bytes("dh_g", lambda ctx: ctx.dh_g_length), + UBInt16("dh_Ys_length"), + Bytes("dh_Ys", lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = Struct( + "pre_master_secret", + ProtocolVersion, + Bytes("random_bytes", 46), +) + +ASN1Cert = Struct( + "ASN1Cert", + UBInt32("length"), # TODO: Reject packets with length not in 1..2^24-1 + Bytes("asn1_cert", lambda ctx: ctx.length), +) + +Certificate = Struct( + "Certificate", # TODO: Reject packets with length > 2 ** 24 - 1 + UBInt32("certificates_length"), + Bytes("certificates_bytes", lambda ctx: ctx.certificates_length), +) + +Handshake = Struct( + "Handshake", + UBInt8("msg_type"), + UBInt24("length"), + Bytes("body", lambda ctx: ctx.length), +) + +Alert = Struct( + "Alert", + UBInt8("level"), + UBInt8("description"), +) diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py new file mode 100644 index 00000000..4c917303 --- /dev/null +++ b/libmproxy/contrib/tls/utils.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import construct + +import six + + +class _UBInt24(construct.Adapter): + def _encode(self, obj, context): + return ( + six.int2byte((obj & 0xFF0000) >> 16) + + six.int2byte((obj & 0x00FF00) >> 8) + + six.int2byte(obj & 0x0000FF) + ) + + def _decode(self, obj, context): + obj = bytearray(obj) + return (obj[0] << 16 | obj[1] << 8 | obj[2]) + + +def UBInt24(name): # noqa + return _UBInt24(construct.Bytes(name, 3)) diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py new file mode 100644 index 00000000..f34d9707 --- /dev/null +++ b/libmproxy/exceptions.py @@ -0,0 +1,34 @@ +from __future__ import (absolute_import, print_function, division) + + +class ProxyException(Exception): + """ + Base class for all exceptions thrown by libmproxy. + """ + def __init__(self, message, cause=None): + """ + :param message: Error Message + :param cause: Exception object that caused this exception to be thrown. + """ + super(ProxyException, self).__init__(message) + self.cause = cause + + +class ProtocolException(ProxyException): + pass + + +class Socks5Exception(ProtocolException): + pass + + +class HttpException(ProtocolException): + pass + + +class InvalidCredentials(HttpException): + pass + + +class ServerException(ProxyException): + pass diff --git a/libmproxy/filt.py b/libmproxy/filt.py index bd17a807..25747bc6 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -246,14 +246,14 @@ class FSrc(_Rex): help = "Match source address" def __call__(self, f): - return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d9ef722..dac607a0 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,6 +8,7 @@ import Cookie import cookielib import os import re +from libmproxy.protocol2.http_replay import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING @@ -859,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: @@ -934,7 +935,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = http.RequestReplayThread( + rt = RequestReplayThread( self.server.config, f, self.masterq if run_scripthooks else False, diff --git a/libmproxy/main.py b/libmproxy/main.py index abf3fb9c..23cb487c 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,11 +2,11 @@ from __future__ import print_function, absolute_import import os import signal import sys -import netlib.version from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline -from .proxy import process_proxy_options, ProxyServerError +from .exceptions import ServerException from .proxy.server import DummyServer, ProxyServer +from .proxy.config import process_proxy_options def assert_utf8_env(): @@ -31,7 +31,7 @@ def get_server(dummy_server, options): else: try: return ProxyServer(options) - except ProxyServerError as v: + except ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..a30437d1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -694,85 +695,4 @@ class HTTPHandler(ProtocolHandler): else: raise http.HttpAuthenticationError( self.c.config.authenticator.auth_challenge_headers()) - return request.headers - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + return request.headers
\ No newline at end of file diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index ed5759ea..b1000a79 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -240,31 +240,15 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +262,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -362,29 +349,15 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request_method, - body_size_limit, - include_body=include_body - ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +366,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py new file mode 100644 index 00000000..61b9a77e --- /dev/null +++ b/libmproxy/protocol2/__init__.py @@ -0,0 +1,13 @@ +from __future__ import (absolute_import, print_function, division) +from .root_context import RootContext +from .socks_proxy import Socks5Proxy +from .reverse_proxy import ReverseProxy +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .transparent_proxy import TransparentProxy +from .http import make_error_response + +__all__ = [ + "RootContext", + "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", + "make_error_response" +] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py new file mode 100644 index 00000000..a508ae8b --- /dev/null +++ b/libmproxy/protocol2/http.py @@ -0,0 +1,588 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import tcp +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from .. import version, utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from .layer import Layer +from ..proxy import Kill +from libmproxy.protocol import KILL, Error +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest + + +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +def make_error_response(status_code, message, headers=None): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. + """ + + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx + + @property + def via(self): + return self._ctx.server_conn + + def __getattr__(self, item): + return getattr(self.via, item) + + +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self.send_request(self.connect_request) + else: + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + +class HttpLayer(Layer): + def __init__(self, ctx, mode): + super(HttpLayer, self).__init__(ctx) + self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" + + def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn + while True: + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + + try: + request = self.read_request() + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + self.handle_regular_mode_connect(request) + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow.request = request + self.process_request_hook(flow) + + if not flow.response: + self.establish_server_connection(flow) + self.get_response_from_server(flow) + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + # TODO: Implement HTTP Upgrade + + # Upstream Proxy Mode: Handle CONNECT + if flow.request.form_in == "authority" and flow.response.code == 200: + self.handle_upstream_mode_connect(flow.request.copy()) + return + + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) + try: + self.send_response(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + finally: + flow.live = False + + def handle_regular_mode_connect(self, request): + self.set_server((request.host, request.port)) + self.send_response(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + layer() + + def handle_upstream_mode_connect(self, connect_request): + layer = UpstreamConnectLayer(self, connect_request) + layer() + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not (self.supports_streaming and flow.response.stream): + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_response(flow.response) + else: + # streaming: + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + self.send_response_body(flow.response, chunks) + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + def get_response(): + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response() + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == KILL: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == KILL: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py new file mode 100644 index 00000000..2876c022 --- /dev/null +++ b/libmproxy/protocol2/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py new file mode 100644 index 00000000..872ef9cd --- /dev/null +++ b/libmproxy/protocol2/http_replay.py @@ -0,0 +1,95 @@ +import threading +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from ..controller import Channel +from ..protocol import KILL, Error +from ..protocol.http_wrappers import HTTPResponse +from ..proxy import Log, Kill +from ..proxy.connection import ServerConnection +from .http import make_connect_request + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py new file mode 100644 index 00000000..2b47cc26 --- /dev/null +++ b/libmproxy/protocol2/layer.py @@ -0,0 +1,138 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + SslLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..proxy import Log +from ..proxy.connection import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py new file mode 100644 index 00000000..b10217f1 --- /dev/null +++ b/libmproxy/protocol2/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .layer import Layer + + +class RawTcpLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTcpLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py new file mode 100644 index 00000000..c4cabccc --- /dev/null +++ b/libmproxy/protocol2/reverse_proxy.py @@ -0,0 +1,18 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .tls import TlsLayer + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py new file mode 100644 index 00000000..210ba6ab --- /dev/null +++ b/libmproxy/protocol2/root_context.py @@ -0,0 +1,94 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from .rawtcp import RawTcpLayer +from .tls import TlsLayer, is_tls_record_magic +from .http import Http1Layer, Http2Layer +from .layer import ServerConnectionMixin +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTcpLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTcpLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py new file mode 100644 index 00000000..525520e8 --- /dev/null +++ b/libmproxy/protocol2/socks_proxy.py @@ -0,0 +1,59 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError +from ..exceptions import Socks5Exception +from .layer import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py new file mode 100644 index 00000000..73bb12f3 --- /dev/null +++ b/libmproxy/protocol2/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .layer import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py new file mode 100644 index 00000000..e6ebf115 --- /dev/null +++ b/libmproxy/protocol2/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from .. import platform +from .layer import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f33d323b..709654cb 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,11 @@ -from .primitives import * -from .config import ProxyConfig, process_proxy_options +from __future__ import (absolute_import, print_function, division) + +from .primitives import Log, Kill +from .config import ProxyConfig +from .connection import ClientConnection, ServerConnection + +__all__ = [ + "Log", "Kill", + "ProxyConfig", + "ClientConnection", "ServerConnection" +]
\ No newline at end of file diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index ec91a6e0..b360abbd 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,26 +1,31 @@ from __future__ import absolute_import +import collections import os import re from OpenSSL import SSL -import netlib -from netlib import http, certutils, tcp +from netlib import certutils, tcp from netlib.http import authentication -from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .. import utils, platform +from netlib.tcp import Address, sslversion_choices -TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] def __call__(self, address): + if not address: + return False address = tcp.Address.wrap(address) host = "%s:%s" % (address.host, address.port) if any(rex.search(host) for rex in self.regexes): @@ -32,61 +37,44 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, host='', port=8080, - server_version=version.NAMEVERSION, cadir=CA_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, - http_form_in=None, - http_form_out=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], - ssl_version_client=tcp.SSL_DEFAULT_METHOD, - ssl_version_server=tcp.SSL_DEFAULT_METHOD, - ssl_ports=TRANSPARENT_SSL_PORTS, - spoofed_ssl_port=None, + certs=tuple(), + ssl_version_client="secure", + ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port - self.server_version = server_version self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - - if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) - elif mode == "socks5": - self.mode = Socks5ProxyMode(ssl_ports) - elif mode == "reverse": - self.mode = ReverseProxyMode(upstream_server) - elif mode == "upstream": - self.mode = UpstreamProxyMode(upstream_server) - elif mode == "spoof": - self.mode = SpoofMode() - elif mode == "sslspoof": - self.mode = SSLSpoofMode(spoofed_ssl_port) + self.mode = mode + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) else: - self.mode = RegularProxyMode() - - # Handle manual overrides of the http forms - self.mode.http_form_in = http_form_in or self.mode.http_form_in - self.mode.http_form_out = http_form_out or self.mode.http_form_out + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -94,41 +82,33 @@ class ProxyConfig: self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( self.cadir, - CONF_BASENAME) + CONF_BASENAME + ) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.ssl_ports = ssl_ports - if isinstance(ssl_version_client, int): - self.openssl_method_client = ssl_version_client - else: - self.openssl_method_client = tcp.SSL_VERSIONS[ssl_version_client] - if isinstance(ssl_version_server, int): - self.openssl_method_server = ssl_version_server - else: - self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - - self.openssl_options_client = tcp.SSL_DEFAULT_OPTIONS - self.openssl_options_server = tcp.SSL_DEFAULT_OPTIONS + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -141,32 +121,26 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -191,12 +165,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -206,99 +174,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=None, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-port", - action="append", - type=int, - dest="ssl_ports", - default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % - str(TRANSPARENT_SSL_PORTS)) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for client connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for server connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + )
\ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 9e03157a..94f318f6 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -10,7 +12,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): # Eventually, this object is restored from state. We don't have a # connection then. if client_connection: - tcp.BaseHandler.__init__(self, client_connection, address, server) + super(ClientConnection, self).__init__(client_connection, address, server) else: self.connection = None self.server = None @@ -25,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): return "<ClientConnection: {ssl}{host}:{port}>".format( ssl="[ssl] " if self.ssl_established else "", @@ -32,6 +37,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( ssl_established=bool, timestamp_start=float, @@ -71,20 +80,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) - - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() def finish(self): - tcp.BaseHandler.finish(self) + super(ClientConnection, self).finish() self.timestamp_end = utils.timestamp() @@ -92,13 +92,16 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.state = [] # a list containing (conntype, state) tuples + self.via = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): if self.ssl_established and self.sni: ssl = "[ssl: {0}] ".format(self.sni) @@ -112,8 +115,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( - state=list, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -131,8 +137,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): d.update( address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= ({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d @@ -176,9 +182,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() @@ -186,3 +189,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def finish(self): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 923f84ca..2e440fe8 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,178 +1,15 @@ from __future__ import absolute_import +import collections from netlib import socks, tcp -class ProxyError(Exception): - def __init__(self, code, message, headers=None): - super(ProxyError, self).__init__(message) - self.code, self.headers = code, headers - - -class ProxyServerError(Exception): - pass - - -class ProxyMode(object): - http_form_in = None - http_form_out = None - - def get_upstream_server(self, client_conn): - """ - Returns the address of the server to connect to. - Returns None if the address needs to be determined on the protocol level (regular proxy mode) - """ - raise NotImplementedError() # pragma: nocover - - @property - def name(self): - return self.__class__.__name__.replace("ProxyMode", "").lower() - - def __str__(self): - return self.name - - def __eq__(self, other): - """ - Allow comparisions with "regular" etc. - """ - if isinstance(other, ProxyMode): - return self is other - else: - return self.name == other - - def __ne__(self, other): - return not self.__eq__(other) - - -class RegularProxyMode(ProxyMode): - http_form_in = "absolute" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - -class SpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "spoof" - - -class SSLSpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslport): - self.sslport = sslport - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "sslspoof" - - -class TransparentProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, resolver, sslports): - self.resolver = resolver - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - dst = self.resolver.original_addr(client_conn.connection) - except Exception as e: - raise ProxyError(502, "Transparent mode failure: %s" % str(e)) - - if dst[1] in self.sslports: - ssl = True - else: - ssl = False - return [ssl, ssl] + list(dst) - - -class Socks5ProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslports): - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(client_conn.wfile) - client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We do not connect here yet, as the clientconnect event has not - # been handled yet. - - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(client_conn.wfile) - client_conn.wfile.flush() - - ssl = bool(connect_request.addr.port in self.sslports) - return ssl, ssl, connect_request.addr.host, connect_request.addr.port - - except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) - - -class _ConstDestinationProxyMode(ProxyMode): - def __init__(self, dst): - self.dst = dst - - def get_upstream_server(self, client_conn): - return self.dst - - -class ReverseProxyMode(_ConstDestinationProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - -class UpstreamProxyMode(_ConstDestinationProxyMode): - http_form_in = "absolute" - http_form_out = "absolute" - - -class Log: +class Log(object): def __init__(self, msg, level="info"): self.msg = msg self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """
\ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2f6ee061..5abd0877 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,13 +1,16 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function +import traceback +import sys import socket -from OpenSSL import SSL - from netlib import tcp -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection -from ..protocol.handle import protocol_handler -from .. import version +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from .. import protocol2 +from ..exceptions import ProtocolException, ServerException +from .primitives import Log, Kill +from .connection import ClientConnection class DummyServer: @@ -33,9 +36,9 @@ class ProxyServer(tcp.TCPServer): """ self.config = config try: - tcp.TCPServer.__init__(self, (config.host, config.port)) - except socket.error as v: - raise ProxyServerError('Error starting proxy server: ' + repr(v)) + super(ProxyServer, self).__init__((config.host, config.port)) + except socket.error as e: + raise ServerException('Error starting proxy server: ' + repr(e), e) self.channel = None def start_slave(self, klass, channel): @@ -47,340 +50,85 @@ class ProxyServer(tcp.TCPServer): def handle_client_connection(self, conn, client_address): h = ConnectionHandler( - self.config, conn, client_address, - self, - self.channel) + self.config, + self.channel + ) h.handle() - h.finish() -class ConnectionHandler: - def __init__( - self, - config, - client_connection, - client_address, - server, - channel): +class ConnectionHandler(object): + def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection( - client_connection, + client_conn, client_address, - server) + None) """@type: libmproxy.proxy.connection.ClientConnection""" - self.server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" self.channel = channel + """@type: libmproxy.controller.Channel""" - self.conntype = "http" + def _create_root_layer(self): + root_context = protocol2.RootContext( + self.client_conn, + self.config, + self.channel + ) + + mode = self.config.mode + if mode == "upstream": + return protocol2.HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + return protocol2.TransparentProxy(root_context) + elif mode == "reverse": + server_tls = self.config.upstream_server.scheme == "https" + return protocol2.ReverseProxy( + root_context, + self.config.upstream_server.address, + server_tls + ) + elif mode == "socks5": + return protocol2.Socks5Proxy(root_context) + elif mode == "regular": + return protocol2.HttpProxy(root_context) + elif callable(mode): # pragma: nocover + return mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) def handle(self): - try: - self.log("clientconnect", "info") - - # Can we already identify the target server and connect to it? - client_ssl, server_ssl = False, False - conn_kwargs = dict() - upstream_info = self.config.mode.get_upstream_server( - self.client_conn) - if upstream_info: - self.set_server_address(upstream_info[2:]) - client_ssl, server_ssl = upstream_info[:2] - if self.config.check_ignore(self.server_conn.address): - self.log( - "Ignore host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - conn_kwargs["log"] = False - client_ssl, server_ssl = False, False - else: - # No upstream info from the metadata: upstream info in the - # protocol (e.g. HTTP absolute-form) - pass - - self.channel.ask("clientconnect", self) - - # Check for existing connection: If an inline script already established a - # connection, do not apply client_ssl or server_ssl. - if self.server_conn and not self.server_conn.connection: - self.establish_server_connection() - if client_ssl or server_ssl: - self.establish_ssl(client=client_ssl, server=server_ssl) - - if self.config.check_tcp(self.server_conn.address): - self.log( - "Generic TCP mode for host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - - elif not self.server_conn and self.config.mode == "sslspoof": - port = self.config.mode.sslport - self.set_server_address(("-", port)) - self.establish_ssl(client=True) - host = self.client_conn.connection.get_servername() - if host: - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) - - # Delegate handling to the protocol handler - protocol_handler( - self.conntype)( - self, - **conn_kwargs).handle_messages() - - self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", self) - - except ProxyError as e: - protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) - except Exception: - import traceback - import sys - - self.log(traceback.format_exc(), "error") - print >> sys.stderr, traceback.format_exc() - print >> sys.stderr, "mitmproxy has crashed!" - print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" - finally: - # Make sure that we close the server connection in any case. - # The client connection is closed by the ProxyServer and does not - # have be handled here. - self.del_server_connection() - - def del_server_connection(self): - """ - Deletes (and closes) an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.server_conn.close() - self.log( - "serverdisconnect", "debug", [ - "%s:%s" % - (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def set_server_address(self, addr): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - """ - address = tcp.Address.wrap(addr) + self.log("clientconnect", "info") - # Don't reconnect to the same destination. - if self.server_conn and self.server_conn.address == address: - return + root_layer = self._create_root_layer() - if self.server_conn: - self.del_server_connection() - - self.log( - "Set new server address: %s:%s" % - (address.host, address.port), "debug") - self.server_conn = ServerConnection(address) - - def establish_server_connection(self, ask=True): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - - By default, this function ".ask"s the proxy master. This is deadly if this function is already called from the - master (e.g. via change_server), because this navigates us in a simple deadlock (the master is single-threaded). - In these scenarios, ask=False can be passed to suppress the call to the master. - """ - if self.server_conn.connection: - return - self.log( - "serverconnect", "debug", [ - "%s:%s" % - self.server_conn.address()[ - :2]]) - if ask: - self.channel.ask("serverconnect", self) try: - self.server_conn.connect() - except tcp.NetLibError as v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False, sni=None): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. - """ - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % sni) - self.log("Establish SSL", "debug", subs) - - if server: - if not self.server_conn or not self.server_conn.connection: - raise ProxyError(502, "No server connection.") - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") + root_layer() + except Kill: + self.log("Connection killed", "info") + except ProtocolException as e: + self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. try: - self.server_conn.establish_ssl( - self.config.clientcerts, - sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibError as v: - e = ProxyError(502, repr(v)) - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/427 - # The upstream server may reject connections without SNI, which means we need to - # establish SSL with the client first, hope for a SNI (which triggers a reconnect which replaces the - # ServerConnection object) and see whether that worked. - if client and "handshake failure" in e.message: - self.server_conn.may_require_sni = e - else: - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise e - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as v: - raise ProxyError(400, repr(v)) - - # Workaround for #427 part 2 - if server and hasattr(self.server_conn, "may_require_sni"): - raise self.server_conn.may_require_sni - - def server_reconnect(self, new_sni=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - state = self.server_conn.state - sni = new_sni or self.server_conn.sni - self.log("(server reconnect follows)", "debug") - self.del_server_connection() - self.set_server_address(address) - self.establish_server_connection() - - for s in state: - protocol_handler(s[0])(self).handle_server_reconnect(s[1]) - self.server_conn.state = state - - # Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for - # https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this - # case, we want to establish SSL as well. - if had_ssl or new_sni: - self.establish_ssl(server=True, sni=sni) + error_response = protocol2.make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass + except Exception: + self.log(traceback.format_exc(), "error") + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) - def finish(self): + self.log("clientdisconnect", "info") self.client_conn.finish() - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if self.server_conn.ssl_established and ( - not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - if self.server_conn.sni: - sans.append(self.server_conn.sni) - # for ssl spoof mode - if hasattr(self.client_conn, "sni"): - sans.append(self.client_conn.sni) - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - # for ssl spoof mode - self.client_conn.sni = sni - - if sni != self.server_conn.sni: - self.log("SNI received: %s" % sni, "debug") - # We should only re-establish upstream SSL if one of the following conditions is true: - # - We established SSL with the server previously - # - We initially wanted to establish SSL with the server, - # but the server refused to negotiate without SNI. - if self.server_conn.ssl_established or hasattr( - self.server_conn, - "may_require_sni"): - # reconnect to upstream server with SNI - self.server_reconnect(sni) - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - import traceback - self.log( - "Error in handle_sni:\r\n" + - traceback.format_exc(), - "error") + def log(self, msg, level): + msg = "{}: {}".format(repr(self.client_conn.address), msg) + self.channel.tell("log", Log(msg, level))
\ No newline at end of file diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 3ac3cc01..a6ca55f7 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,14 +1,10 @@ from __future__ import absolute_import import os import datetime -import urllib import re import time -import functools -import cgi import json -import netlib.utils def timestamp(): """ |