diff options
| -rw-r--r-- | doc-src/features/reverseproxy.html | 2 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 79 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 25 | ||||
| -rw-r--r-- | libmproxy/console/help.py | 1 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 58 | ||||
| -rw-r--r-- | libmproxy/proxy/primitives.py | 41 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 42 | ||||
| -rw-r--r-- | libmproxy/utils.py | 9 | ||||
| -rw-r--r-- | test/test_cmdline.py | 11 | ||||
| -rw-r--r-- | test/test_proxy.py | 26 | ||||
| -rw-r--r-- | test/test_utils.py | 8 | ||||
| -rw-r--r-- | test/tservers.py | 31 | ||||
| -rw-r--r-- | test/tutils.py | 11 |
14 files changed, 182 insertions, 164 deletions
diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index d399cdc0..7be02b87 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -10,7 +10,7 @@ __Host__ header field from the request, not the reverse proxy server. <table class="table"> <tbody> <tr> - <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td> + <th width="20%">command-line</th> <td>-R http[s]://hostname[:port]</td> </tr> <tr> <th>mitmproxy shortcut</th> <td><b>P</b></td> diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 72c13769..6c0cae9f 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,12 +1,16 @@ from . import proxy import re, filt import argparse +from argparse import ArgumentTypeError +from netlib import http APP_HOST = "mitm.it" APP_PORT = 80 -class ParseException(Exception): pass -class OptionException(Exception): pass + +class ParseException(Exception): + pass + def _parse_hook(s): sep, rem = s[0], s[1:] @@ -91,6 +95,26 @@ def parse_setheader(s): return _parse_hook(s) +def parse_server_spec(url): + + normalized_url = re.sub("^https?2", "", url) + + p = http.parse_url(normalized_url) + if not p or not p[1]: + raise ArgumentTypeError("Invalid server specification: %s" % url) + + if url.lower().startswith("https2http"): + ssl = [True, False] + elif url.lower().startswith("http2https"): + ssl = [False, True] + elif url.lower().startswith("https"): + ssl = [True, True] + else: + ssl = [False, False] + + return ssl + list(p[1:3]) + + def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -104,17 +128,17 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise OptionException("Could not read replace file: %s"%path) + raise ArgumentTypeError("Could not read replace file: %s"%path) reps.append((patt, rex, v)) @@ -123,7 +147,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -185,17 +209,49 @@ def common_options(parser): action="store", type = int, dest="port", default=8080, help = "Proxy service port." ) + # We could make a mutually exclusive group out of -R, -F, -T, but we don't do because + # - --upstream-server should be in that group as well, but it's already in a different group. + # - our own error messages are more helpful parser.add_argument( - "-P", - action="store", dest="reverse_proxy", default=None, + "-R", + action="store", type=parse_server_spec, dest="reverse_proxy", default=None, help="Reverse proxy to upstream server: http[s]://host[:port]" ) parser.add_argument( "-F", - action="store", dest="forward_proxy", default=None, + action="store", type=parse_server_spec, dest="forward_proxy", default=None, help="Proxy to unconditionally forward to: http[s]://host[:port]" ) parser.add_argument( + "-T", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + + group = parser.add_argument_group( + "Advanced Proxy Options", + """ + The following options allow a custom adjustment of the proxy behavior. + Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -F, -T). + """.strip() + ) + group.add_argument( + "--http-form-in", dest="http_form_in", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form accepted by the proxy" + ) + group.add_argument( + "--http-form-out", dest="http_form_out", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form sent upstream by the proxy" + ) + group.add_argument( + "--upstream-server", dest="manual_upstream_server", default=None, + action="store", type=parse_server_spec, + help="Override the destination server all requests are sent to." + ) + + parser.add_argument( "-q", action="store_true", dest="quiet", help="Quiet." @@ -217,11 +273,6 @@ def common_options(parser): help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( - "-T", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." - ) - parser.add_argument( "-u", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", help="Set sticky auth filter. Matched against requests." diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4a58e771..846abba7 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -144,10 +144,6 @@ class StatusBar(common.WWrap): r.append("[") r.append(("heading_key", "u")) r.append(":%s]"%self.master.stickyauth_txt) - if self.master.server.config.reverse_proxy: - r.append("[") - r.append(("heading_key", "P")) - r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy)) if self.master.state.default_body_view.name != "Auto": r.append("[") r.append(("heading_key", "M")) @@ -172,6 +168,8 @@ class StatusBar(common.WWrap): if opts: r.append("[%s]"%(":".join(opts))) + if self.master.server.config.upstream_server: + r.append("[dest:%s]"%utils.unparse_url(*self.master.server.config.upstream_server)) if self.master.scripts: r.append("[scripts:%s]"%len(self.master.scripts)) if self.master.debug: @@ -763,15 +761,6 @@ class ConsoleMaster(flow.FlowMaster): self.state.default_body_view = v self.refresh_focus() - def set_reverse_proxy(self, txt): - if not txt: - self.server.config.reverse_proxy = None - else: - s = utils.parse_proxy_spec(txt) - if not s: - return "Invalid reverse proxy specification" - self.server.config.reverse_proxy = s - def drawscreen(self): size = self.ui.get_cols_rows() canvas = self.view.render(size, focus=1) @@ -866,16 +855,6 @@ class ConsoleMaster(flow.FlowMaster): contentview.view_prompts, self.change_default_display_mode ) - elif k == "P": - if self.server.config.reverse_proxy: - p = utils.unparse_url(*self.server.config.reverse_proxy) - else: - p = "" - self.prompt( - "Reverse proxy: ", - p, - self.set_reverse_proxy - ) elif k == "R": self.view_grideditor( grideditor.ReplaceEditor( diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 0d01ac6f..b0b7c48c 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -109,7 +109,6 @@ class HelpView(urwid.ListBox): ("q", "quit / return to flow list"), ("Q", "quit without confirm prompt"), - ("P", "set reverse proxy mode"), ("R", "edit replacement patterns"), ("s", "set/unset script"), ("S", "server replay"), diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index c8147a3c..aff4a817 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1027,7 +1027,7 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): raise http.HttpError(400, "Must not CONNECT on already encrypted connection") if self.expected_form_in == "absolute": - if not self.c.config.upstream_server: + if not self.c.config.get_upstream_server: self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow self.c.client_conn.send( diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 248808e9..ae24d4c9 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,7 +1,7 @@ import os from .. import utils, platform from netlib import http_auth, certutils - +from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -10,18 +10,17 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: def __init__(self, confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, upstream_server=None, - http_form_in="absolute", http_form_out="relative", transparent_proxy=None, authenticator=None, + no_upstream_cert=False, body_size_limit=None, get_upstream_server=None, + http_form_in="absolute", http_form_out="relative", authenticator=None, ciphers=None, certs=None ): self.ciphers = ciphers self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - self.upstream_server = upstream_server + self.get_upstream_server = get_upstream_server self.http_form_in = http_form_in self.http_form_out = http_form_out - self.transparent_proxy = transparent_proxy self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) @@ -29,32 +28,34 @@ class ProxyConfig: def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - return parser.error("Can't set both reverse proxy and transparent proxy.") + c = 0 + http_form_in, http_form_out = "absolute", "relative" + get_upstream_server = None if options.transparent_proxy: + c += 1 if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") - trans = dict( - resolver=platform.resolver(), - sslports=TRANSPARENT_SSL_PORTS - ) - else: - trans = None - + get_upstream_server = TransparentUpstreamServerResolver(platform.resolver(), TRANSPARENT_SSL_PORTS) + http_form_in, http_form_out = "relative", "relative" if options.reverse_proxy: - rp = utils.parse_proxy_spec(options.reverse_proxy) - if not rp: - return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy) - else: - rp = None - + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.reverse_proxy) + http_form_in, http_form_out = "relative", "relative" if options.forward_proxy: - fp = utils.parse_proxy_spec(options.forward_proxy) - if not fp: - return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy) - else: - fp = None + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.forward_proxy) + http_form_in, http_form_out = "absolute", "absolute" + if options.manual_upstream_server: + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.manual_upstream_server) + if c > 1: + return parser.error("Transparent mode, reverse mode, forward mode and " + "specification of an upstream server are mutually exclusive.") + if options.http_form_in: + http_form_in = options.http_form_in + if options.http_form_out: + http_form_out = options.http_form_out if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) @@ -94,10 +95,9 @@ def process_proxy_options(parser, options): clientcerts=options.clientcerts, body_size_limit=body_size_limit, no_upstream_cert=options.no_upstream_cert, - upstream_server=(rp or fp), - http_form_in=("relative" if (rp or trans) else "absolute"), - http_form_out=("absolute" if fp else "relative"), - transparent_proxy=trans, + get_upstream_server=get_upstream_server, + http_form_in=http_form_in, + http_form_out=http_form_out, authenticator=authenticator, ciphers=options.ciphers, certs = certs, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 75ad5482..e49d9bb4 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -18,17 +18,48 @@ class ProxyServerError(Exception): pass +class UpstreamServerResolver(object): + def __call__(self, conn): + """ + Returns the address of the server to connect to. + """ + raise NotImplementedError + + +class ConstUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, dst): + self.dst = dst + + def __call__(self, conn): + return self.dst + + +class TransparentUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, resolver, sslports): + self.resolver = resolver + self.sslports = sslports + + def __call__(self, conn): + dst = self.resolver.original_addr(conn) + if not dst: + raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + + if dst[1] in self.sslports: + ssl = True + else: + ssl = False + return [ssl, ssl] + list(dst) + + class AddressPriority(object): """ Enum that signifies the priority of the given address when choosing the destination host. Higher is better (None < i) """ - MANUALLY_CHANGED = 4 + MANUALLY_CHANGED = 3 """user changed the target address in the ui""" - FROM_SETTINGS = 3 - """upstream proxy from arguments (reverse proxy or forward proxy)""" - FROM_CONNECTION = 2 - """derived from transparent resolver""" + FROM_SETTINGS = 2 + """upstream server from arguments (reverse proxy, forward proxy or from transparent resolver)""" FROM_PROTOCOL = 1 """derived from protocol (e.g. absolute-form http requests)""" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5aaabf87..554ee551 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -68,22 +68,13 @@ class ConnectionHandler: try: try: # Can we already identify the target server and connect to it? - server_address = None - address_priority = None - if self.config.upstream_server: - server_address = self.config.upstream_server[1:] - address_priority = AddressPriority.FROM_SETTINGS - elif self.config.transparent_proxy: - server_address = self.config.transparent_proxy["resolver"].original_addr( - self.client_conn.connection) - if not server_address: - raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") - address_priority = AddressPriority.FROM_CONNECTION - self.log("transparent to %s:%s" % server_address) - - if server_address: - self.set_server_address(server_address, address_priority) - self._handle_ssl() + if self.config.get_upstream_server: + upstream_info = self.config.get_upstream_server(self.client_conn.connection) + self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS) + client_ssl, server_ssl = upstream_info[:2] + if client_ssl or server_ssl: + self.establish_server_connection() + self.establish_ssl(client=client_ssl, server=server_ssl) while not self.close: try: @@ -105,25 +96,6 @@ class ConnectionHandler: self.log("clientdisconnect") self.channel.tell("clientdisconnect", self) - def _handle_ssl(self): - """ - Helper function of .handle() - Check if we can already identify SSL connections. - If so, connect to the server and establish an SSL connection - """ - client_ssl = False - server_ssl = False - - if self.config.transparent_proxy: - client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) - elif self.config.upstream_server: - client_ssl = server_ssl = (self.config.upstream_server[0] == "https") - # TODO: Make protocol generic (as with transparent proxies) - # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) - if client_ssl or server_ssl: - self.establish_server_connection() - self.establish_ssl(client=client_ssl, server=server_ssl) - def del_server_connection(self): """ Deletes an existing server connection. diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 1ec4685d..17ad6f09 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,7 +1,6 @@ import os, datetime, urllib, re import time, functools, cgi import json -from netlib import http def timestamp(): """ @@ -143,14 +142,6 @@ class LRUCache: return ret return wrap - -def parse_proxy_spec(url): - p = http.parse_url(url) - if not p or not p[1]: - return None - return p[:3] - - def parse_content_type(c): """ A simple parser for content-type values. Returns a (type, subtype, diff --git a/test/test_cmdline.py b/test/test_cmdline.py index dbc61bfc..12e8aa89 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -36,6 +36,17 @@ def test_parse_replace_hook(): ) +def test_parse_server_spec(): + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") + assert cmdline.parse_server_spec("http://foo.com:88") == [False, False, "foo.com", 88] + assert cmdline.parse_server_spec("http://foo.com") == [False, False, "foo.com", 80] + assert cmdline.parse_server_spec("https://foo.com") == [True, True, "foo.com", 443] + assert cmdline.parse_server_spec("https2http://foo.com") == [True, False, "foo.com", 80] + assert cmdline.parse_server_spec("http2https://foo.com") == [False, True, "foo.com", 443] + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "foo.com") + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "http://") + + def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") diff --git a/test/test_proxy.py b/test/test_proxy.py index f53aa762..0f438482 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -43,28 +43,15 @@ class TestServerConnection: sc.finish() -class MockParser: - def __init__(self): - self.err = None - - def error(self, e): - self.err = e - - def __repr__(self): - return "ParseError(%s)"%self.err - - class TestProcessProxyOptions: def p(self, *args): - parser = argparse.ArgumentParser() + parser = tutils.MockParser() cmdline.common_options(parser) opts = parser.parse_args(args=args) - m = MockParser() - return m, process_proxy_options(m, opts) + return parser, process_proxy_options(parser, opts) def assert_err(self, err, *args): - m, p = self.p(*args) - assert err.lower() in m.err.lower() + tutils.raises(err, self.p, *args) def assert_noerr(self, *args): m, p = self.p(*args) @@ -84,11 +71,10 @@ class TestProcessProxyOptions: @mock.patch("libmproxy.platform.resolver") def test_transparent_reverse(self, o): - self.assert_err("can't set both", "-P", "reverse", "-T") + self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") self.assert_noerr("-T") - assert o.call_count == 1 - self.assert_err("invalid reverse proxy", "-P", "reverse") - self.assert_noerr("-P", "http://localhost") + self.assert_err("Invalid server specification", "-R", "reverse") + self.assert_noerr("-R", "http://localhost") def test_client_certs(self): with tutils.tmpdir() as confdir: diff --git a/test/test_utils.py b/test/test_utils.py index f40e0a31..11c26dba 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -87,14 +87,6 @@ def test_LRUCache(): assert len(f._cachelist_one) == 2 -def test_parse_proxy_spec(): - assert not utils.parse_proxy_spec("") - assert utils.parse_proxy_spec("http://foo.com:88") == ("http", "foo.com", 88) - assert utils.parse_proxy_spec("http://foo.com") == ("http", "foo.com", 80) - assert not utils.parse_proxy_spec("foo.com") - assert not utils.parse_proxy_spec("http://") - - def test_unparse_url(): assert utils.unparse_url("http", "foo.com", 99, "") == "http://foo.com:99" assert utils.unparse_url("http", "foo.com", 80, "") == "http://foo.com" diff --git a/test/tservers.py b/test/tservers.py index addc7011..9ad3b735 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -4,6 +4,7 @@ import shutil, tempfile import flask from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer +from libmproxy.proxy.primitives import TransparentUpstreamServerResolver import libpathod.test, libpathod.pathoc from libmproxy import flow, controller from libmproxy.cmdline import APP_HOST, APP_PORT @@ -193,10 +194,7 @@ class TransparentProxTest(ProxTestBase): ports = [cls.server.port, cls.server2.port] else: ports = [] - d["transparent_proxy"] = dict( - resolver = cls.resolver(cls.server.port), - sslports = ports - ) + d["get_upstream_server"] = TransparentUpstreamServerResolver(cls.resolver(cls.server.port), ports) d["http_form_in"] = "relative" d["http_form_out"] = "relative" return d @@ -227,11 +225,12 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = ( - "https" if cls.ssl else "http", - "127.0.0.1", - cls.server.port - ) + d["get_upstream_server"] = lambda c: ( + True if cls.ssl else False, + True if cls.ssl else False, + "127.0.0.1", + cls.server.port + ) d["http_form_in"] = "relative" d["http_form_out"] = "relative" return d @@ -262,19 +261,17 @@ class ChainProxTest(ProxTestBase): Chain n instances of mitmproxy in a row - because we can. """ n = 2 - chain_config = [lambda: ProxyConfig()] * n + chain_config = [lambda port: ProxyConfig( + get_upstream_server = lambda c: (False, False, "127.0.0.1", port), + http_form_in = "absolute", + http_form_out = "absolute" + )] * n @classmethod def setupAll(cls): super(ChainProxTest, cls).setupAll() cls.chain = [] for i in range(cls.n): - config = cls.chain_config[i]() - config.upstream_server = ("http", "127.0.0.1", - cls.proxy.port if i == 0 else - cls.chain[-1].port - ) - config.http_form_in = "absolute" - config.http_form_out = "absolute" + config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port) tmaster = cls.masterclass(config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.chain.append(ProxyThread(tmaster)) diff --git a/test/tutils.py b/test/tutils.py index 3f6592b0..c527a64a 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -1,4 +1,4 @@ -import os, shutil, tempfile +import os, shutil, tempfile, argparse from contextlib import contextmanager from libmproxy import flow, utils, controller from libmproxy.protocol import http @@ -136,6 +136,15 @@ def tmpdir(*args, **kwargs): shutil.rmtree(temp_workdir) +class MockParser(argparse.ArgumentParser): + """ + argparse.ArgumentParser sys.exits() by default. + Make it more testable by throwing an exception instead. + """ + def error(self, message): + raise Exception(message) + + def raises(exc, obj, *args, **kwargs): """ Assert that a callable raises a specified exception. |