diff options
39 files changed, 2127 insertions, 1133 deletions
diff --git a/doc-src/howmitmproxy.html b/doc-src/howmitmproxy.html index fabd393a..16b5f722 100644 --- a/doc-src/howmitmproxy.html +++ b/doc-src/howmitmproxy.html @@ -145,15 +145,6 @@ passed to us. Now we can pause the conversation, and initiate an upstream connection using the correct SNI value, which then serves us the correct upstream certificate, from which we can extract the expected CN and SANs. -There's another wrinkle here. Due to a limitation of the SSL library mitmproxy -uses, we can't detect that a connection _hasn't_ sent an SNI request until it's -too late for upstream certificate sniffing. In practice, we therefore make a -vanilla SSL connection upstream to sniff non-SNI certificates, and then discard -the connection if the client sends an SNI notification. If you're watching your -traffic with a packet sniffer, you'll see two connections to the server when an -SNI request is made, the first of which is immediately closed after the SSL -handshake. Luckily, this is almost never an issue in practice. - ## Putting it all together Lets put all of this together into the complete explicitly proxied HTTPS flow. diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 6125bfbf..7f6f69ef 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address, sslversion_choices -from netlib import http import netlib.utils from . import filt, utils, version @@ -108,26 +108,9 @@ def parse_server_spec(url): raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) - - -def parse_server_spec_special(url): - """ - Provides additional support for http2https and https2http schemes. - """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def get_common_options(options): @@ -192,24 +175,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +284,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +358,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ - ) + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) - - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ + group.add_argument( + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) + +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +455,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +464,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +530,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +548,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +573,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +597,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +625,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -633,20 +675,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -663,13 +705,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/console/statusbar.py b/libmproxy/console/statusbar.py index 7eb2131b..ea2dbfa8 100644 --- a/libmproxy/console/statusbar.py +++ b/libmproxy/console/statusbar.py @@ -199,11 +199,12 @@ class StatusBar(urwid.WidgetWrap): r.append("[%s]" % (":".join(opts))) if self.master.server.config.mode in ["reverse", "upstream"]: - dst = self.master.server.config.mode.dst - scheme = "https" if dst[0] else "http" - if dst[1] != dst[0]: - scheme += "2https" if dst[1] else "http" - r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:])) + dst = self.master.server.config.upstream_server + r.append("[dest:%s]" % netlib.utils.unparse_url( + dst.scheme, + dst.address.host, + dst.address.port + )) if self.master.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index 3b0f7512..e5ce11da 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -8,3 +8,7 @@ jsbeautifier, git checkout 25/03/12, MIT license wbxml - https://github.com/davidpshaw/PyWBXMLDecoder + +tls, BSD license + - https://github.com/mhils/tls/tree/mitmproxy + - limited to required files.
\ No newline at end of file diff --git a/libmproxy/contrib/tls/__init__.py b/libmproxy/contrib/tls/__init__.py new file mode 100644 index 00000000..4b540884 --- /dev/null +++ b/libmproxy/contrib/tls/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py new file mode 100644 index 00000000..9c57a799 --- /dev/null +++ b/libmproxy/contrib/tls/_constructs.py @@ -0,0 +1,213 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from construct import (Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, TunnelAdapter, GreedyRange, + Switch, OptionalGreedyRange) + +from .utils import UBInt24 + +ProtocolVersion = Struct( + "version", + UBInt8("major"), + UBInt8("minor"), +) + +TLSPlaintext = Struct( + "TLSPlaintext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCompressed = Struct( + "TLSCompressed", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 1024 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCiphertext = Struct( + "TLSCiphertext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 2048 + Bytes("fragment", lambda ctx: ctx.length), +) + +Random = Struct( + "random", + UBInt32("gmt_unix_time"), + Bytes("random_bytes", 28), +) + +SessionID = Struct( + "session_id", + UBInt8("length"), + Bytes("session_id", lambda ctx: ctx.length), +) + +CipherSuites = Struct( + "cipher_suites", + UBInt16("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, UBInt16("cipher_suites")), +) + +CompressionMethods = Struct( + "compression_methods", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("compression_methods")), +) + +ServerName = Struct( + "", + UBInt8("type"), + PascalString("name", length_field=UBInt16("length")), +) + +SNIExtension = Struct( + "", + TunnelAdapter( + PascalString("server_names", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(ServerName) + ), + ), +) + +ALPNExtension = Struct( + "", + TunnelAdapter( + PascalString("alpn_protocols", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(PascalString("name")) + ), + ), +) + +UnknownExtension = Struct( + "", + PascalString("bytes", length_field=UBInt16("extensions_length")) +) + +Extension = Struct( + "Extension", + UBInt16("type"), + Embed( + Switch( + "", lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension + }, + default=UnknownExtension + ) + ) +) + +extensions = TunnelAdapter( + PascalString("extensions", length_field=UBInt16("extensions_length")), + OptionalGreedyRange(Extension) +) + +ClientHello = Struct( + "ClientHello", + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = Struct( + "ServerHello", + ProtocolVersion, + Random, + SessionID, + Bytes("cipher_suite", 2), + UBInt8("compression_method"), + extensions, +) + +ClientCertificateType = Struct( + "certificate_types", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("certificate_types")), +) + +SignatureAndHashAlgorithm = Struct( + "algorithms", + UBInt8("hash"), + UBInt8("signature"), +) + +SupportedSignatureAlgorithms = Struct( + "supported_signature_algorithms", + UBInt16("supported_signature_algorithms_length"), + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = Struct( + "certificate_authorities", + UBInt16("length"), + Bytes("certificate_authorities", lambda ctx: ctx.length), +) + +CertificateRequest = Struct( + "CertificateRequest", + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = Struct( + "ServerDHParams", + UBInt16("dh_p_length"), + Bytes("dh_p", lambda ctx: ctx.dh_p_length), + UBInt16("dh_g_length"), + Bytes("dh_g", lambda ctx: ctx.dh_g_length), + UBInt16("dh_Ys_length"), + Bytes("dh_Ys", lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = Struct( + "pre_master_secret", + ProtocolVersion, + Bytes("random_bytes", 46), +) + +ASN1Cert = Struct( + "ASN1Cert", + UBInt32("length"), # TODO: Reject packets with length not in 1..2^24-1 + Bytes("asn1_cert", lambda ctx: ctx.length), +) + +Certificate = Struct( + "Certificate", # TODO: Reject packets with length > 2 ** 24 - 1 + UBInt32("certificates_length"), + Bytes("certificates_bytes", lambda ctx: ctx.certificates_length), +) + +Handshake = Struct( + "Handshake", + UBInt8("msg_type"), + UBInt24("length"), + Bytes("body", lambda ctx: ctx.length), +) + +Alert = Struct( + "Alert", + UBInt8("level"), + UBInt8("description"), +) diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py new file mode 100644 index 00000000..4c917303 --- /dev/null +++ b/libmproxy/contrib/tls/utils.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import construct + +import six + + +class _UBInt24(construct.Adapter): + def _encode(self, obj, context): + return ( + six.int2byte((obj & 0xFF0000) >> 16) + + six.int2byte((obj & 0x00FF00) >> 8) + + six.int2byte(obj & 0x0000FF) + ) + + def _decode(self, obj, context): + obj = bytearray(obj) + return (obj[0] << 16 | obj[1] << 8 | obj[2]) + + +def UBInt24(name): # noqa + return _UBInt24(construct.Bytes(name, 3)) diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py new file mode 100644 index 00000000..f34d9707 --- /dev/null +++ b/libmproxy/exceptions.py @@ -0,0 +1,34 @@ +from __future__ import (absolute_import, print_function, division) + + +class ProxyException(Exception): + """ + Base class for all exceptions thrown by libmproxy. + """ + def __init__(self, message, cause=None): + """ + :param message: Error Message + :param cause: Exception object that caused this exception to be thrown. + """ + super(ProxyException, self).__init__(message) + self.cause = cause + + +class ProtocolException(ProxyException): + pass + + +class Socks5Exception(ProtocolException): + pass + + +class HttpException(ProtocolException): + pass + + +class InvalidCredentials(HttpException): + pass + + +class ServerException(ProxyException): + pass diff --git a/libmproxy/filt.py b/libmproxy/filt.py index bd17a807..25747bc6 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -246,14 +246,14 @@ class FSrc(_Rex): help = "Match source address" def __call__(self, f): - return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d9ef722..dac607a0 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,6 +8,7 @@ import Cookie import cookielib import os import re +from libmproxy.protocol2.http_replay import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING @@ -859,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: @@ -934,7 +935,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = http.RequestReplayThread( + rt = RequestReplayThread( self.server.config, f, self.masterq if run_scripthooks else False, diff --git a/libmproxy/main.py b/libmproxy/main.py index abf3fb9c..23cb487c 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,11 +2,11 @@ from __future__ import print_function, absolute_import import os import signal import sys -import netlib.version from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline -from .proxy import process_proxy_options, ProxyServerError +from .exceptions import ServerException from .proxy.server import DummyServer, ProxyServer +from .proxy.config import process_proxy_options def assert_utf8_env(): @@ -31,7 +31,7 @@ def get_server(dummy_server, options): else: try: return ProxyServer(options) - except ProxyServerError as v: + except ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..a30437d1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -694,85 +695,4 @@ class HTTPHandler(ProtocolHandler): else: raise http.HttpAuthenticationError( self.c.config.authenticator.auth_challenge_headers()) - return request.headers - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + return request.headers
\ No newline at end of file diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index ed5759ea..b1000a79 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -240,31 +240,15 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +262,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -362,29 +349,15 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request_method, - body_size_limit, - include_body=include_body - ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +366,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py new file mode 100644 index 00000000..61b9a77e --- /dev/null +++ b/libmproxy/protocol2/__init__.py @@ -0,0 +1,13 @@ +from __future__ import (absolute_import, print_function, division) +from .root_context import RootContext +from .socks_proxy import Socks5Proxy +from .reverse_proxy import ReverseProxy +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .transparent_proxy import TransparentProxy +from .http import make_error_response + +__all__ = [ + "RootContext", + "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", + "make_error_response" +] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py new file mode 100644 index 00000000..a508ae8b --- /dev/null +++ b/libmproxy/protocol2/http.py @@ -0,0 +1,588 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import tcp +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from .. import version, utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from .layer import Layer +from ..proxy import Kill +from libmproxy.protocol import KILL, Error +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest + + +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +def make_error_response(status_code, message, headers=None): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. + """ + + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx + + @property + def via(self): + return self._ctx.server_conn + + def __getattr__(self, item): + return getattr(self.via, item) + + +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self.send_request(self.connect_request) + else: + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + +class HttpLayer(Layer): + def __init__(self, ctx, mode): + super(HttpLayer, self).__init__(ctx) + self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" + + def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn + while True: + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + + try: + request = self.read_request() + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + self.handle_regular_mode_connect(request) + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow.request = request + self.process_request_hook(flow) + + if not flow.response: + self.establish_server_connection(flow) + self.get_response_from_server(flow) + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + # TODO: Implement HTTP Upgrade + + # Upstream Proxy Mode: Handle CONNECT + if flow.request.form_in == "authority" and flow.response.code == 200: + self.handle_upstream_mode_connect(flow.request.copy()) + return + + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) + try: + self.send_response(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + finally: + flow.live = False + + def handle_regular_mode_connect(self, request): + self.set_server((request.host, request.port)) + self.send_response(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + layer() + + def handle_upstream_mode_connect(self, connect_request): + layer = UpstreamConnectLayer(self, connect_request) + layer() + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not (self.supports_streaming and flow.response.stream): + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_response(flow.response) + else: + # streaming: + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + self.send_response_body(flow.response, chunks) + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + def get_response(): + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response() + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == KILL: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == KILL: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py new file mode 100644 index 00000000..2876c022 --- /dev/null +++ b/libmproxy/protocol2/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py new file mode 100644 index 00000000..872ef9cd --- /dev/null +++ b/libmproxy/protocol2/http_replay.py @@ -0,0 +1,95 @@ +import threading +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from ..controller import Channel +from ..protocol import KILL, Error +from ..protocol.http_wrappers import HTTPResponse +from ..proxy import Log, Kill +from ..proxy.connection import ServerConnection +from .http import make_connect_request + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py new file mode 100644 index 00000000..2b47cc26 --- /dev/null +++ b/libmproxy/protocol2/layer.py @@ -0,0 +1,138 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + SslLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..proxy import Log +from ..proxy.connection import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py new file mode 100644 index 00000000..b10217f1 --- /dev/null +++ b/libmproxy/protocol2/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .layer import Layer + + +class RawTcpLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTcpLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py new file mode 100644 index 00000000..c4cabccc --- /dev/null +++ b/libmproxy/protocol2/reverse_proxy.py @@ -0,0 +1,18 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .tls import TlsLayer + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py new file mode 100644 index 00000000..210ba6ab --- /dev/null +++ b/libmproxy/protocol2/root_context.py @@ -0,0 +1,94 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from .rawtcp import RawTcpLayer +from .tls import TlsLayer, is_tls_record_magic +from .http import Http1Layer, Http2Layer +from .layer import ServerConnectionMixin +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTcpLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTcpLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py new file mode 100644 index 00000000..525520e8 --- /dev/null +++ b/libmproxy/protocol2/socks_proxy.py @@ -0,0 +1,59 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError +from ..exceptions import Socks5Exception +from .layer import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py new file mode 100644 index 00000000..73bb12f3 --- /dev/null +++ b/libmproxy/protocol2/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .layer import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py new file mode 100644 index 00000000..e6ebf115 --- /dev/null +++ b/libmproxy/protocol2/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from .. import platform +from .layer import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f33d323b..709654cb 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,11 @@ -from .primitives import * -from .config import ProxyConfig, process_proxy_options +from __future__ import (absolute_import, print_function, division) + +from .primitives import Log, Kill +from .config import ProxyConfig +from .connection import ClientConnection, ServerConnection + +__all__ = [ + "Log", "Kill", + "ProxyConfig", + "ClientConnection", "ServerConnection" +]
\ No newline at end of file diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index ec91a6e0..b360abbd 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,26 +1,31 @@ from __future__ import absolute_import +import collections import os import re from OpenSSL import SSL -import netlib -from netlib import http, certutils, tcp +from netlib import certutils, tcp from netlib.http import authentication -from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .. import utils, platform +from netlib.tcp import Address, sslversion_choices -TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] def __call__(self, address): + if not address: + return False address = tcp.Address.wrap(address) host = "%s:%s" % (address.host, address.port) if any(rex.search(host) for rex in self.regexes): @@ -32,61 +37,44 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, host='', port=8080, - server_version=version.NAMEVERSION, cadir=CA_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, - http_form_in=None, - http_form_out=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], - ssl_version_client=tcp.SSL_DEFAULT_METHOD, - ssl_version_server=tcp.SSL_DEFAULT_METHOD, - ssl_ports=TRANSPARENT_SSL_PORTS, - spoofed_ssl_port=None, + certs=tuple(), + ssl_version_client="secure", + ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port - self.server_version = server_version self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - - if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) - elif mode == "socks5": - self.mode = Socks5ProxyMode(ssl_ports) - elif mode == "reverse": - self.mode = ReverseProxyMode(upstream_server) - elif mode == "upstream": - self.mode = UpstreamProxyMode(upstream_server) - elif mode == "spoof": - self.mode = SpoofMode() - elif mode == "sslspoof": - self.mode = SSLSpoofMode(spoofed_ssl_port) + self.mode = mode + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) else: - self.mode = RegularProxyMode() - - # Handle manual overrides of the http forms - self.mode.http_form_in = http_form_in or self.mode.http_form_in - self.mode.http_form_out = http_form_out or self.mode.http_form_out + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -94,41 +82,33 @@ class ProxyConfig: self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( self.cadir, - CONF_BASENAME) + CONF_BASENAME + ) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.ssl_ports = ssl_ports - if isinstance(ssl_version_client, int): - self.openssl_method_client = ssl_version_client - else: - self.openssl_method_client = tcp.SSL_VERSIONS[ssl_version_client] - if isinstance(ssl_version_server, int): - self.openssl_method_server = ssl_version_server - else: - self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - - self.openssl_options_client = tcp.SSL_DEFAULT_OPTIONS - self.openssl_options_server = tcp.SSL_DEFAULT_OPTIONS + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -141,32 +121,26 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -191,12 +165,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -206,99 +174,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=None, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-port", - action="append", - type=int, - dest="ssl_ports", - default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % - str(TRANSPARENT_SSL_PORTS)) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for client connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for server connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + )
\ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 9e03157a..94f318f6 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -10,7 +12,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): # Eventually, this object is restored from state. We don't have a # connection then. if client_connection: - tcp.BaseHandler.__init__(self, client_connection, address, server) + super(ClientConnection, self).__init__(client_connection, address, server) else: self.connection = None self.server = None @@ -25,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): return "<ClientConnection: {ssl}{host}:{port}>".format( ssl="[ssl] " if self.ssl_established else "", @@ -32,6 +37,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( ssl_established=bool, timestamp_start=float, @@ -71,20 +80,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) - - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() def finish(self): - tcp.BaseHandler.finish(self) + super(ClientConnection, self).finish() self.timestamp_end = utils.timestamp() @@ -92,13 +92,16 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.state = [] # a list containing (conntype, state) tuples + self.via = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): if self.ssl_established and self.sni: ssl = "[ssl: {0}] ".format(self.sni) @@ -112,8 +115,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( - state=list, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -131,8 +137,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): d.update( address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= ({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d @@ -176,9 +182,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() @@ -186,3 +189,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def finish(self): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 923f84ca..2e440fe8 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,178 +1,15 @@ from __future__ import absolute_import +import collections from netlib import socks, tcp -class ProxyError(Exception): - def __init__(self, code, message, headers=None): - super(ProxyError, self).__init__(message) - self.code, self.headers = code, headers - - -class ProxyServerError(Exception): - pass - - -class ProxyMode(object): - http_form_in = None - http_form_out = None - - def get_upstream_server(self, client_conn): - """ - Returns the address of the server to connect to. - Returns None if the address needs to be determined on the protocol level (regular proxy mode) - """ - raise NotImplementedError() # pragma: nocover - - @property - def name(self): - return self.__class__.__name__.replace("ProxyMode", "").lower() - - def __str__(self): - return self.name - - def __eq__(self, other): - """ - Allow comparisions with "regular" etc. - """ - if isinstance(other, ProxyMode): - return self is other - else: - return self.name == other - - def __ne__(self, other): - return not self.__eq__(other) - - -class RegularProxyMode(ProxyMode): - http_form_in = "absolute" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - -class SpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "spoof" - - -class SSLSpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslport): - self.sslport = sslport - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "sslspoof" - - -class TransparentProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, resolver, sslports): - self.resolver = resolver - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - dst = self.resolver.original_addr(client_conn.connection) - except Exception as e: - raise ProxyError(502, "Transparent mode failure: %s" % str(e)) - - if dst[1] in self.sslports: - ssl = True - else: - ssl = False - return [ssl, ssl] + list(dst) - - -class Socks5ProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslports): - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(client_conn.wfile) - client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We do not connect here yet, as the clientconnect event has not - # been handled yet. - - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(client_conn.wfile) - client_conn.wfile.flush() - - ssl = bool(connect_request.addr.port in self.sslports) - return ssl, ssl, connect_request.addr.host, connect_request.addr.port - - except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) - - -class _ConstDestinationProxyMode(ProxyMode): - def __init__(self, dst): - self.dst = dst - - def get_upstream_server(self, client_conn): - return self.dst - - -class ReverseProxyMode(_ConstDestinationProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - -class UpstreamProxyMode(_ConstDestinationProxyMode): - http_form_in = "absolute" - http_form_out = "absolute" - - -class Log: +class Log(object): def __init__(self, msg, level="info"): self.msg = msg self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """
\ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2f6ee061..5abd0877 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,13 +1,16 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function +import traceback +import sys import socket -from OpenSSL import SSL - from netlib import tcp -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection -from ..protocol.handle import protocol_handler -from .. import version +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from .. import protocol2 +from ..exceptions import ProtocolException, ServerException +from .primitives import Log, Kill +from .connection import ClientConnection class DummyServer: @@ -33,9 +36,9 @@ class ProxyServer(tcp.TCPServer): """ self.config = config try: - tcp.TCPServer.__init__(self, (config.host, config.port)) - except socket.error as v: - raise ProxyServerError('Error starting proxy server: ' + repr(v)) + super(ProxyServer, self).__init__((config.host, config.port)) + except socket.error as e: + raise ServerException('Error starting proxy server: ' + repr(e), e) self.channel = None def start_slave(self, klass, channel): @@ -47,340 +50,85 @@ class ProxyServer(tcp.TCPServer): def handle_client_connection(self, conn, client_address): h = ConnectionHandler( - self.config, conn, client_address, - self, - self.channel) + self.config, + self.channel + ) h.handle() - h.finish() -class ConnectionHandler: - def __init__( - self, - config, - client_connection, - client_address, - server, - channel): +class ConnectionHandler(object): + def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection( - client_connection, + client_conn, client_address, - server) + None) """@type: libmproxy.proxy.connection.ClientConnection""" - self.server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" self.channel = channel + """@type: libmproxy.controller.Channel""" - self.conntype = "http" + def _create_root_layer(self): + root_context = protocol2.RootContext( + self.client_conn, + self.config, + self.channel + ) + + mode = self.config.mode + if mode == "upstream": + return protocol2.HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + return protocol2.TransparentProxy(root_context) + elif mode == "reverse": + server_tls = self.config.upstream_server.scheme == "https" + return protocol2.ReverseProxy( + root_context, + self.config.upstream_server.address, + server_tls + ) + elif mode == "socks5": + return protocol2.Socks5Proxy(root_context) + elif mode == "regular": + return protocol2.HttpProxy(root_context) + elif callable(mode): # pragma: nocover + return mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) def handle(self): - try: - self.log("clientconnect", "info") - - # Can we already identify the target server and connect to it? - client_ssl, server_ssl = False, False - conn_kwargs = dict() - upstream_info = self.config.mode.get_upstream_server( - self.client_conn) - if upstream_info: - self.set_server_address(upstream_info[2:]) - client_ssl, server_ssl = upstream_info[:2] - if self.config.check_ignore(self.server_conn.address): - self.log( - "Ignore host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - conn_kwargs["log"] = False - client_ssl, server_ssl = False, False - else: - # No upstream info from the metadata: upstream info in the - # protocol (e.g. HTTP absolute-form) - pass - - self.channel.ask("clientconnect", self) - - # Check for existing connection: If an inline script already established a - # connection, do not apply client_ssl or server_ssl. - if self.server_conn and not self.server_conn.connection: - self.establish_server_connection() - if client_ssl or server_ssl: - self.establish_ssl(client=client_ssl, server=server_ssl) - - if self.config.check_tcp(self.server_conn.address): - self.log( - "Generic TCP mode for host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - - elif not self.server_conn and self.config.mode == "sslspoof": - port = self.config.mode.sslport - self.set_server_address(("-", port)) - self.establish_ssl(client=True) - host = self.client_conn.connection.get_servername() - if host: - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) - - # Delegate handling to the protocol handler - protocol_handler( - self.conntype)( - self, - **conn_kwargs).handle_messages() - - self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", self) - - except ProxyError as e: - protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) - except Exception: - import traceback - import sys - - self.log(traceback.format_exc(), "error") - print >> sys.stderr, traceback.format_exc() - print >> sys.stderr, "mitmproxy has crashed!" - print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" - finally: - # Make sure that we close the server connection in any case. - # The client connection is closed by the ProxyServer and does not - # have be handled here. - self.del_server_connection() - - def del_server_connection(self): - """ - Deletes (and closes) an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.server_conn.close() - self.log( - "serverdisconnect", "debug", [ - "%s:%s" % - (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def set_server_address(self, addr): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - """ - address = tcp.Address.wrap(addr) + self.log("clientconnect", "info") - # Don't reconnect to the same destination. - if self.server_conn and self.server_conn.address == address: - return + root_layer = self._create_root_layer() - if self.server_conn: - self.del_server_connection() - - self.log( - "Set new server address: %s:%s" % - (address.host, address.port), "debug") - self.server_conn = ServerConnection(address) - - def establish_server_connection(self, ask=True): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - - By default, this function ".ask"s the proxy master. This is deadly if this function is already called from the - master (e.g. via change_server), because this navigates us in a simple deadlock (the master is single-threaded). - In these scenarios, ask=False can be passed to suppress the call to the master. - """ - if self.server_conn.connection: - return - self.log( - "serverconnect", "debug", [ - "%s:%s" % - self.server_conn.address()[ - :2]]) - if ask: - self.channel.ask("serverconnect", self) try: - self.server_conn.connect() - except tcp.NetLibError as v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False, sni=None): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. - """ - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % sni) - self.log("Establish SSL", "debug", subs) - - if server: - if not self.server_conn or not self.server_conn.connection: - raise ProxyError(502, "No server connection.") - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") + root_layer() + except Kill: + self.log("Connection killed", "info") + except ProtocolException as e: + self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. try: - self.server_conn.establish_ssl( - self.config.clientcerts, - sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibError as v: - e = ProxyError(502, repr(v)) - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/427 - # The upstream server may reject connections without SNI, which means we need to - # establish SSL with the client first, hope for a SNI (which triggers a reconnect which replaces the - # ServerConnection object) and see whether that worked. - if client and "handshake failure" in e.message: - self.server_conn.may_require_sni = e - else: - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise e - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as v: - raise ProxyError(400, repr(v)) - - # Workaround for #427 part 2 - if server and hasattr(self.server_conn, "may_require_sni"): - raise self.server_conn.may_require_sni - - def server_reconnect(self, new_sni=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - state = self.server_conn.state - sni = new_sni or self.server_conn.sni - self.log("(server reconnect follows)", "debug") - self.del_server_connection() - self.set_server_address(address) - self.establish_server_connection() - - for s in state: - protocol_handler(s[0])(self).handle_server_reconnect(s[1]) - self.server_conn.state = state - - # Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for - # https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this - # case, we want to establish SSL as well. - if had_ssl or new_sni: - self.establish_ssl(server=True, sni=sni) + error_response = protocol2.make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass + except Exception: + self.log(traceback.format_exc(), "error") + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) - def finish(self): + self.log("clientdisconnect", "info") self.client_conn.finish() - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if self.server_conn.ssl_established and ( - not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - if self.server_conn.sni: - sans.append(self.server_conn.sni) - # for ssl spoof mode - if hasattr(self.client_conn, "sni"): - sans.append(self.client_conn.sni) - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - # for ssl spoof mode - self.client_conn.sni = sni - - if sni != self.server_conn.sni: - self.log("SNI received: %s" % sni, "debug") - # We should only re-establish upstream SSL if one of the following conditions is true: - # - We established SSL with the server previously - # - We initially wanted to establish SSL with the server, - # but the server refused to negotiate without SNI. - if self.server_conn.ssl_established or hasattr( - self.server_conn, - "may_require_sni"): - # reconnect to upstream server with SNI - self.server_reconnect(sni) - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - import traceback - self.log( - "Error in handle_sni:\r\n" + - traceback.format_exc(), - "error") + def log(self, msg, level): + msg = "{}: {}".format(repr(self.client_conn.address), msg) + self.channel.tell("log", Log(msg, level))
\ No newline at end of file diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 3ac3cc01..a6ca55f7 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,14 +1,10 @@ from __future__ import absolute_import import os import datetime -import urllib import re import time -import functools -import cgi import json -import netlib.utils def timestamp(): """ @@ -20,7 +20,9 @@ deps = { "pyperclip>=1.5.8", "blinker>=1.3", "pyparsing>=1.5.2", - "html2text>=2015.4.14" + "html2text>=2015.4.14", + "construct>=2.5.2", + "six>=1.9.0", } # A script -> additional dependencies dict. scripts = { diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py index e5c323be..e26d83f1 100644 --- a/test/scripts/stream_modify.py +++ b/test/scripts/stream_modify.py @@ -1,6 +1,6 @@ def modify(chunks): - for prefix, content, suffix in chunks: - yield prefix, content.replace("foo", "bar"), suffix + for chunk in chunks: + yield chunk.replace("foo", "bar") def responseheaders(context, flow): diff --git a/test/test_cmdline.py b/test/test_cmdline.py index eafcbde4..1443ee1c 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -38,15 +38,11 @@ def test_parse_replace_hook(): def test_parse_server_spec(): tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") assert cmdline.parse_server_spec( - "http://foo.com:88") == [False, False, "foo.com", 88] + "http://foo.com:88") == ("http", ("foo.com", 88)) assert cmdline.parse_server_spec( - "http://foo.com") == [False, False, "foo.com", 80] + "http://foo.com") == ("http", ("foo.com", 80)) assert cmdline.parse_server_spec( - "https://foo.com") == [True, True, "foo.com", 443] - assert cmdline.parse_server_spec_special( - "https2http://foo.com") == [True, False, "foo.com", 80] - assert cmdline.parse_server_spec_special( - "http2https://foo.com") == [False, True, "foo.com", 443] + "https://foo.com") == ("https", ("foo.com", 443)) tutils.raises( "Invalid server specification", cmdline.parse_server_spec, @@ -55,6 +51,10 @@ def test_parse_server_spec(): "Invalid server specification", cmdline.parse_server_spec, "http://") + tutils.raises( + "Invalid server specification", + cmdline.parse_server_spec, + "https2http://foo.com") def test_parse_setheaders(): diff --git a/test/test_dump.py b/test/test_dump.py index b3d724a5..b05f6a0f 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -5,8 +5,8 @@ import netlib.tutils from netlib.http.semantics import CONTENT_MISSING from libmproxy import dump, flow -from libmproxy.protocol import http, http_wrappers -from libmproxy.proxy.primitives import Log +from libmproxy.protocol import http_wrappers +from libmproxy.proxy import Log import tutils import mock diff --git a/test/test_flow.py b/test/test_flow.py index 711688da..5c49deed 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -4,6 +4,7 @@ import os.path from cStringIO import StringIO import email.utils import mock +from libmproxy.cmdline import parse_server_spec import netlib.utils from netlib import odict @@ -672,11 +673,8 @@ class TestSerialize: s = flow.State() conf = ProxyConfig( mode="reverse", - upstream_server=[ - True, - True, - "use-this-domain", - 80]) + upstream_server=("https", ("use-this-domain", 80)) + ) fm = flow.FlowMaster(DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 2da54093..cd0f77fa 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -4,6 +4,7 @@ from cStringIO import StringIO from mock import MagicMock from libmproxy.protocol.http import * +import netlib.http from netlib import odict from netlib.http import http1 from netlib.http.semantics import CONTENT_MISSING @@ -56,7 +57,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) assert r.status_code == 400 - assert "Must not CONNECT on already encrypted connection" in r.body + assert "Invalid HTTP request form" in r.body def test_relative_request(self): p = self.pathoc_raw() diff --git a/test/test_proxy.py b/test/test_proxy.py index 6ab19e02..301ce2ca 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,8 +1,7 @@ -import argparse from libmproxy import cmdline -from libmproxy.proxy import ProxyConfig, process_proxy_options +from libmproxy.proxy import ProxyConfig +from libmproxy.proxy.config import process_proxy_options from libmproxy.proxy.connection import ServerConnection -from libmproxy.proxy.primitives import ProxyError from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler import tutils from libpathod import test @@ -12,11 +11,6 @@ import mock from OpenSSL import SSL -def test_proxy_error(): - p = ProxyError(111, "msg") - assert str(p) - - class TestServerConnection: def setUp(self): self.d = test.Daemon() @@ -97,13 +91,7 @@ class TestProcessProxyOptions: self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") - self.assert_noerr("--spoof") - self.assert_noerr("--ssl-spoof") - - self.assert_noerr("--spoofed-port", "443") - self.assert_err("expected one argument", "--spoofed-port") - - self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") + self.assert_err("not allowed with", "-R", "http://localhost", "-T") def test_client_certs(self): with tutils.tmpdir() as cadir: @@ -181,13 +169,14 @@ class TestDummyServer: class TestConnectionHandler: def test_fatal_error(self): config = mock.Mock() - config.mode.get_upstream_server.side_effect = RuntimeError + root_layer = mock.Mock() + root_layer.side_effect = RuntimeError + config.mode.return_value = root_layer c = ConnectionHandler( - config, mock.MagicMock(), - ("127.0.0.1", - 8080), - None, - mock.MagicMock()) + ("127.0.0.1", 8080), + config, + mock.MagicMock() + ) with tutils.capture_stderr(c.handle) as output: assert "mitmproxy has crashed" in output diff --git a/test/test_server.py b/test/test_server.py index 77ba4576..b691804b 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,6 +1,7 @@ import socket import time from OpenSSL import SSL +from netlib.tcp import Address import netlib.tutils from netlib import tcp, http, socks @@ -67,7 +68,7 @@ class CommonMixin: # SSL with the upstream proxy. rt = self.master.replay_request(l, block=True) assert not rt - if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl: + if isinstance(self, tservers.HTTPUpstreamProxTest): assert l.response.code == 502 else: assert l.error @@ -319,17 +320,6 @@ class TestHTTPAuth(tservers.HTTPProxTest): assert ret.status_code == 202 -class TestHTTPConnectSSLError(tservers.HTTPProxTest): - certfile = True - - def test_go(self): - self.config.ssl_ports.append(self.proxy.port) - p = self.pathoc_raw() - dst = ("localhost", self.proxy.port) - p.connect(connect_to=dst) - tutils.raises("502 - Bad Gateway", p.http_connect, dst) - - class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin): ssl = True ssloptions = pathod.SSLOptions(request_client_cert=True) @@ -390,26 +380,31 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxTest): ("untrusted-cert", tutils.test_data.path("data/untrusted-server.crt")) ]) + def _request(self): + p = self.pathoc() + # We need to make an actual request because the upstream connection is lazy-loaded. + return p.request("get:/p/242") + def test_default_verification_w_bad_cert(self): """Should use no verification.""" self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - self.pathoc() + assert self._request().status_code == 242 def test_no_verification_w_bad_cert(self): self.config.openssl_verification_mode_server = SSL.VERIFY_NONE self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - self.pathoc() + assert self._request().status_code == 242 def test_verification_w_bad_cert(self): self.config.openssl_verification_mode_server = SSL.VERIFY_PEER self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - tutils.raises("SSL handshake error", self.pathoc) + assert self._request().status_code == 502 class TestHTTPSNoCommonName(tservers.HTTPProxTest): @@ -469,60 +464,11 @@ class TestSocks5(tservers.SocksModeTest): assert "SOCKS5 mode failure" in f.content -class TestSpoof(tservers.SpoofModeTest): - def test_http(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - p = self.pathoc() - f = p.request("get:/p/304:h'Host'='%s:%s'" % a) - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_http_without_host(self): - p = self.pathoc() - f = p.request("get:/p/304:r") - assert f.status_code == 400 - - -class TestSSLSpoof(tservers.SSLSpoofModeTest): - def test_https(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - self.config.mode.sslport = a[1] - p = self.pathoc(sni=a[0]) - f = p.request("get:/p/304") - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_https_without_sni(self): - a = ("localhost", self.server.port) - self.config.mode.sslport = a[1] - p = self.pathoc(sni=None) - f = p.request("get:/p/304") - assert f.status_code == 400 - - class TestHttps2Http(tservers.ReverseProxTest): @classmethod def get_proxy_config(cls): d = super(TestHttps2Http, cls).get_proxy_config() - d["upstream_server"][0] = True + d["upstream_server"] = ("https2http", d["upstream_server"][1]) return d def pathoc(self, ssl, sni=None): @@ -544,10 +490,6 @@ class TestHttps2Http(tservers.ReverseProxTest): assert p.request("get:'/p/200'").status_code == 200 assert all("Error in handle_sni" not in msg for msg in self.proxy.log) - def test_http(self): - p = self.pathoc(ssl=False) - assert p.request("get:'/p/200'").status_code == 400 - class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False @@ -560,7 +502,7 @@ class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin): p = pathoc.Pathoc(("localhost", self.proxy.port), fp=None) p.connect() r = p.request("get:/") - assert r.status_code == 400 + assert r.status_code == 502 class TestProxy(tservers.HTTPProxTest): @@ -661,63 +603,67 @@ class MasterRedirectRequest(tservers.TestMaster): redirect_port = None # Set by TestRedirectRequest def handle_request(self, f): - request = f.request - if request.path == "/p/201": - addr = f.live.c.server_conn.address - assert f.live.change_server( - ("127.0.0.1", self.redirect_port), ssl=False) - assert not f.live.change_server( - ("127.0.0.1", self.redirect_port), ssl=False) - tutils.raises( - "SSL handshake error", - f.live.change_server, - ("127.0.0.1", - self.redirect_port), - ssl=True) - assert f.live.change_server(addr, ssl=False) - request.url = "http://127.0.0.1:%s/p/201" % self.redirect_port - tservers.TestMaster.handle_request(self, f) + if f.request.path == "/p/201": + + # This part should have no impact, but it should not cause any exceptions. + addr = f.live.server_conn.address + addr2 = Address(("127.0.0.1", self.redirect_port)) + f.live.set_server(addr2) + f.live.connect() + f.live.set_server(addr) + f.live.connect() + + # This is the actual redirection. + f.request.port = self.redirect_port + super(MasterRedirectRequest, self).handle_request(f) def handle_response(self, f): f.response.content = str(f.client_conn.address.port) f.response.headers[ "server-conn-id"] = [str(f.server_conn.source_address.port)] - tservers.TestMaster.handle_response(self, f) + super(MasterRedirectRequest, self).handle_response(f) class TestRedirectRequest(tservers.HTTPProxTest): masterclass = MasterRedirectRequest + ssl = True def test_redirect(self): + """ + Imagine a single HTTPS connection with three requests: + + 1. First request should pass through unmodified + 2. Second request will be redirected to a different host by an inline script + 3. Third request should pass through unmodified + + This test verifies that the original destination is restored for the third request. + """ self.master.redirect_port = self.server2.port p = self.pathoc() self.server.clear_log() self.server2.clear_log() - r1 = p.request("get:'%s/p/200'" % self.server.urlbase) + r1 = p.request("get:'/p/200'") assert r1.status_code == 200 assert self.server.last_log() assert not self.server2.last_log() self.server.clear_log() self.server2.clear_log() - r2 = p.request("get:'%s/p/201'" % self.server.urlbase) + r2 = p.request("get:'/p/201'") assert r2.status_code == 201 assert not self.server.last_log() assert self.server2.last_log() self.server.clear_log() self.server2.clear_log() - r3 = p.request("get:'%s/p/202'" % self.server.urlbase) + r3 = p.request("get:'/p/202'") assert r3.status_code == 202 assert self.server.last_log() assert not self.server2.last_log() assert r1.content == r2.content == r3.content - assert r1.headers.get_first( - "server-conn-id") == r3.headers.get_first("server-conn-id") - # Make sure that we actually use the same connection in this test case class MasterStreamRequest(tservers.TestMaster): @@ -774,9 +720,9 @@ class TestStreamRequest(tservers.HTTPProxTest): assert resp.headers["Transfer-Encoding"][0] == 'chunked' assert resp.status_code == 200 - chunks = list( - content for _, content, _ in protocol.read_http_body_chunked( - resp.headers, None, "GET", 200, False)) + chunks = list(protocol.read_http_body_chunked( + resp.headers, None, "GET", 200, False + )) assert chunks == ["this", "isatest", ""] connection.close() @@ -1009,6 +955,9 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): p = self.pathoc() req = p.request("get:'/p/418:b\"content\"'") + assert req.content == "content" + assert req.status_code == 418 + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request # CONNECT, failing request, assert self.chain[0].tmaster.state.flow_count() == 4 @@ -1017,8 +966,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): assert self.chain[1].tmaster.state.flow_count() == 2 # (doesn't store (repeated) CONNECTs from chain[0] # as it is a regular proxy) - assert req.content == "content" - assert req.status_code == 418 + assert not self.chain[1].tmaster.state.flows[0].response # killed assert self.chain[1].tmaster.state.flows[1].response diff --git a/test/tservers.py b/test/tservers.py index 3c73b262..c5256e53 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -1,6 +1,5 @@ import os.path import threading -import Queue import shutil import tempfile import flask @@ -8,7 +7,6 @@ import mock from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer -from libmproxy.proxy.primitives import TransparentProxyMode import libpathod.test import libpathod.pathoc from libmproxy import flow, controller @@ -130,7 +128,6 @@ class ProxTestBase(object): no_upstream_cert = cls.no_upstream_cert, cadir = cls.cadir, authenticator = cls.authenticator, - ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None ) @@ -183,22 +180,24 @@ class TResolver: def original_addr(self, sock): return ("127.0.0.1", self.port) - class TransparentProxTest(ProxTestBase): ssl = None resolver = TResolver @classmethod - @mock.patch("libmproxy.platform.resolver") - def setupAll(cls, _): + def setupAll(cls): super(TransparentProxTest, cls).setupAll() - if cls.ssl: - ports = [cls.server.port, cls.server2.port] - else: - ports = [] - cls.config.mode = TransparentProxyMode( - cls.resolver(cls.server.port), - ports) + + cls._resolver = mock.patch( + "libmproxy.platform.resolver", + new=lambda: cls.resolver(cls.server.port) + ) + cls._resolver.start() + + @classmethod + def teardownAll(cls): + cls._resolver.stop() + super(TransparentProxTest, cls).teardownAll() @classmethod def get_proxy_config(cls): @@ -235,12 +234,10 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = [ - True if cls.ssl else False, - True if cls.ssl else False, - "127.0.0.1", - cls.server.port - ] + d["upstream_server"] = ( + "https" if cls.ssl else "http", + ("127.0.0.1", cls.server.port) + ) d["mode"] = "reverse" return d @@ -274,48 +271,6 @@ class SocksModeTest(HTTPProxTest): d["mode"] = "socks5" return d -class SpoofModeTest(ProxTestBase): - ssl = None - - @classmethod - def get_proxy_config(cls): - d = ProxTestBase.get_proxy_config() - d["upstream_server"] = None - d["mode"] = "spoof" - return d - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - p.connect() - return p - - -class SSLSpoofModeTest(ProxTestBase): - ssl = True - - @classmethod - def get_proxy_config(cls): - d = ProxTestBase.get_proxy_config() - d["upstream_server"] = None - d["mode"] = "sslspoof" - d["spoofed_ssl_port"] = 443 - return d - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - p.connect() - return p - class ChainProxTest(ProxTestBase): """ @@ -360,7 +315,7 @@ class ChainProxTest(ProxTestBase): if cls.chain: # First proxy is in normal mode. d.update( mode="upstream", - upstream_server=(False, False, "127.0.0.1", cls.chain[0].port) + upstream_server=("http", ("127.0.0.1", cls.chain[0].port)) ) return d |