From ece15b3c8af790e425ebcaa6807d6e133d810ff6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 8 Sep 2014 14:43:05 +0200 Subject: reverse proxy: adjust dst when reading flows, fix #346 --- libmproxy/flow.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index d263ccdd..6aa26f4a 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -615,6 +615,11 @@ class FlowMaster(controller.Master): """ Loads a flow, and returns a new flow object. """ + + if self.server and self.server.config.mode == "reverse": + f.request.host, f.request.port = self.server.config.mode.dst[2:] + f.request.scheme = "https" if self.server.config.mode.dst[1] else "http" + f.reply = controller.DummyReply() if f.request: self.handle_request(f) -- cgit v1.2.3 From 6ce6b1ad69df886af6da025c16d4d6916a56da2c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 2 Oct 2014 00:58:40 +0200 Subject: replay: carry over SNI value --- libmproxy/protocol/http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index de5f9950..0bb014a2 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1304,7 +1304,7 @@ class RequestReplayThread(threading.Thread): server.connect() if r.scheme == "https": send_connect_request(server, r.host, r.port) - server.establish_ssl(self.config.clientcerts, sni=r.host) + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) r.form_out = "relative" else: r.form_out = "absolute" @@ -1313,10 +1313,11 @@ class RequestReplayThread(threading.Thread): server = ServerConnection(server_address) server.connect() if r.scheme == "https": - server.establish_ssl(self.config.clientcerts, sni=r.host) + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) r.form_out = "relative" server.send(r.assemble()) + self.flow.server_conn = server self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) self.channel.ask("response", self.flow) -- cgit v1.2.3 From d5c318b070305ac51e6b37f80336ab471af28d26 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Oct 2014 20:44:52 +0200 Subject: fix support for chained certificates --- libmproxy/proxy/config.py | 4 ++-- libmproxy/proxy/server.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 62104a24..24e09b6a 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -16,7 +16,7 @@ def parse_host_pattern(patterns): class ProxyConfig: def __init__(self, host='', port=8080, server_version=version.NAMEVERSION, - confdir=CONF_DIR, ca_file=None, clientcerts=None, + confdir=CONF_DIR, default_ca=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, authenticator=None, ignore=[], @@ -45,7 +45,7 @@ class ProxyConfig: self.ignore = parse_host_pattern(ignore) self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) - self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") + self.default_ca = default_ca or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 307a4bcd..0152f539 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -190,14 +190,14 @@ class ConnectionHandler: if client: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() try: self.client_conn.convert_to_ssl( cert, key, handle_sni=self.handle_sni, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) except tcp.NetLibError as v: raise ProxyError(400, repr(v)) @@ -264,17 +264,17 @@ class ConnectionHandler: self.log("SNI received: %s" % self.sni, "debug") self.server_reconnect() # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( cert, key, method=SSL.TLSv1_METHOD, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. - except Exception: # pragma: no cover + except: # pragma: no cover import traceback self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") \ No newline at end of file -- cgit v1.2.3 From f04693c04779b6c78d0370c0ffd15f899b9b522f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Oct 2014 21:41:03 +0200 Subject: fix typo --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 0bb014a2..32a88b4b 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1079,7 +1079,7 @@ class HTTPHandler(ProtocolHandler): if message: self.c.log(message, level="info") if message_debug: - self.c.log(message, level="debug") + self.c.log(message_debug, level="debug") if flow: # TODO: no flows without request or with both request and response at the moment. -- cgit v1.2.3 From 5b33f7896136012ab8cd86999f5af2b90e66125b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Oct 2014 00:49:11 +0200 Subject: add mini documentation --- libmproxy/proxy/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 24e09b6a..b5974807 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -133,10 +133,12 @@ def ssl_option_group(parser): group.add_argument( "--cert", dest='certs', default=[], type=str, metavar="SPEC", action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' \ - 'The domain may include a wildcard, and is equal to "*" if not specified. ' \ - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' \ - 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' + 'it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' + 'Can be passed multiple times.' ) group.add_argument( "--client-certs", action="store", -- cgit v1.2.3 From 7c56a3bb019f521fc45953923b94e9249a1fca78 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Oct 2014 01:58:54 +0200 Subject: Add SOCKS5 mode, fix #305 --- libmproxy/cmdline.py | 8 +++-- libmproxy/proxy/config.py | 9 +++-- libmproxy/proxy/primitives.py | 81 +++++++++++++++++++++++++++++++++++++++---- libmproxy/proxy/server.py | 2 +- 4 files changed, 89 insertions(+), 11 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index f6cd1ab8..fe68e95e 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,6 +1,5 @@ from __future__ import absolute_import import re -import argparse from argparse import ArgumentTypeError from netlib import http from . import filt, utils @@ -287,6 +286,11 @@ def common_options(parser): action="store", type=parse_server_spec, dest="reverse_proxy", default=None, help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]" ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) group.add_argument( "-T", action="store_true", dest="transparent_proxy", default=False, @@ -381,7 +385,7 @@ def common_options(parser): action="append", dest="replay_ignore_params", type=str, help="Request's parameters to be ignored while searching for a saved flow to replay" "Can be passed multiple times." - ) + ) group = parser.add_argument_group( "Replacements", diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index b5974807..e641546f 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -3,7 +3,7 @@ import os import re from netlib import http_auth, certutils from .. import utils, platform, version -from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode +from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -31,6 +31,8 @@ class ProxyConfig: if mode == "transparent": self.mode = TransparentProxyMode(platform.resolver(), TRANSPARENT_SSL_PORTS) + elif mode == "socks5": + self.mode = Socks5ProxyMode(TRANSPARENT_SSL_PORTS) elif mode == "reverse": self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": @@ -63,6 +65,9 @@ def process_proxy_options(parser, options): if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") mode = "transparent" + if options.socks_proxy: + c += 1 + mode = "socks5" if options.reverse_proxy: c += 1 mode = "reverse" @@ -72,7 +77,7 @@ def process_proxy_options(parser, options): mode = "upstream" upstream_server = options.upstream_proxy if c > 1: - return parser.error("Transparent mode, reverse mode and upstream proxy mode " + return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode " "are mutually exclusive.") if options.clientcerts: diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 23d089d3..c0ae424d 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,5 +1,5 @@ from __future__ import absolute_import - +from netlib import socks class ProxyError(Exception): def __init__(self, code, message, headers=None): @@ -15,7 +15,7 @@ class ProxyMode(object): http_form_in = None http_form_out = None - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): """ Returns the address of the server to connect to. Returns None if the address needs to be determined on the protocol level (regular proxy mode) @@ -46,7 +46,7 @@ class RegularProxyMode(ProxyMode): http_form_in = "absolute" http_form_out = "relative" - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return None @@ -58,9 +58,9 @@ class TransparentProxyMode(ProxyMode): self.resolver = resolver self.sslports = sslports - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): try: - dst = self.resolver.original_addr(conn) + dst = self.resolver.original_addr(client_conn.connection) except Exception, e: raise ProxyError(502, "Transparent mode failure: %s" % str(e)) @@ -71,11 +71,80 @@ class TransparentProxyMode(ProxyMode): return [ssl, ssl] + list(dst) +class Socks5ProxyMode(ProxyMode): + http_form_in = "relative" + http_form_out = "relative" + + def __init__(self, sslports): + self.sslports = sslports + + @staticmethod + def _assert_socks5(msg): + if msg.ver != socks.VERSION.SOCKS5: + if msg.ver == ord("G") and len(msg.methods) == ord("E"): + guess = "Probably not a SOCKS request but a regular HTTP request. " + else: + guess = "" + raise socks.SocksError( + socks.REP.GENERAL_SOCKS_SERVER_FAILURE, + guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver) + + def get_upstream_server(self, client_conn): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(client_conn.rfile) + self._assert_socks5(client_greet) + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(client_conn.wfile) + client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(client_conn.rfile) + self._assert_socks5(connect_request) + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We do not connect here yet, as the clientconnect event has not been handled yet. + + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + socks.ATYP.DOMAINNAME, + client_conn.address # dummy value, we don't have an upstream connection yet. + ) + connect_reply.to_file(client_conn.wfile) + client_conn.wfile.flush() + + ssl = bool(connect_request.addr.port in self.sslports) + return ssl, ssl, connect_request.addr.host, connect_request.addr.port + + except socks.SocksError as e: + msg = socks.Message(5, e.code, socks.ATYP.DOMAINNAME, repr(e)) + try: + msg.to_file(client_conn.wfile) + except: + pass + raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) + + class _ConstDestinationProxyMode(ProxyMode): def __init__(self, dst): self.dst = dst - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return self.dst diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 0152f539..57932b0f 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -70,7 +70,7 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? client_ssl, server_ssl = False, False - upstream_info = self.config.mode.get_upstream_server(self.client_conn.connection) + upstream_info = self.config.mode.get_upstream_server(self.client_conn) if upstream_info: self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] -- cgit v1.2.3 From d0809a210b8269ffc8e3e6808403b934671e625e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Oct 2014 02:47:32 +0200 Subject: fix cert forwarding --- libmproxy/proxy/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 57932b0f..4c7fbbf0 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -234,7 +234,7 @@ class ConnectionHandler: def find_cert(self): if self.config.certforward and self.server_conn.ssl_established: - return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert) + return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None else: host = self.server_conn.address.host sans = [] -- cgit v1.2.3 From 52b29d49264e1397db6c65ee773479391b3fd37a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 18 Oct 2014 15:26:10 +0200 Subject: remove default_ca --- libmproxy/cmdline.py | 2 +- libmproxy/proxy/config.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index fe68e95e..c0eb57c9 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -189,7 +189,7 @@ def common_options(parser): parser.add_argument( "--confdir", action="store", type=str, dest="confdir", default='~/.mitmproxy', - help="Configuration directory. (~/.mitmproxy)" + help="Configuration directory, contains default CA file. (~/.mitmproxy)" ) parser.add_argument( "--host", diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index e641546f..abdb7c41 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -16,7 +16,7 @@ def parse_host_pattern(patterns): class ProxyConfig: def __init__(self, host='', port=8080, server_version=version.NAMEVERSION, - confdir=CONF_DIR, default_ca=None, clientcerts=None, + confdir=CONF_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, authenticator=None, ignore=[], @@ -47,7 +47,6 @@ class ProxyConfig: self.ignore = parse_host_pattern(ignore) self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) - self.default_ca = default_ca or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) -- cgit v1.2.3 From e1148584380058f264b7aa7e9493115e4e8f2bbe Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 18 Oct 2014 18:29:35 +0200 Subject: add generic tcp proxying, fix #374 --- libmproxy/cmdline.py | 9 ++++++++- libmproxy/console/__init__.py | 30 +++++++++++++++++++++++------- libmproxy/console/grideditor.py | 4 ++-- libmproxy/console/help.py | 1 + libmproxy/flow.py | 18 ++++++++++++------ libmproxy/protocol/http.py | 9 +++++++-- libmproxy/protocol/primitives.py | 6 +++--- libmproxy/protocol/tcp.py | 35 +++++++++++++++++++---------------- libmproxy/proxy/config.py | 27 +++++++++++++++++++++------ libmproxy/proxy/server.py | 20 +++++++++----------- libmproxy/web/static/flows.json | 28 ++++++++++++++-------------- 11 files changed, 119 insertions(+), 68 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index c0eb57c9..83eab7ee 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -263,7 +263,7 @@ def common_options(parser): ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore", default=[], + action="append", type=str, dest="ignore_hosts", default=[], metavar="HOST", help="Ignore host and forward all traffic without processing it. " "In transparent mode, it is recommended to use an IP address (range), not the hostname. " @@ -271,6 +271,13 @@ def common_options(parser): "The supplied value is interpreted as a regular expression and matched on the ip or the hostname. " "Can be passed multiple times. " ) + group.add_argument( + "--tcp", + action="append", type=str, dest="tcp_hosts", default=[], + metavar="HOST", + help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore," + "but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode." + ) group.add_argument( "-n", action="store_true", dest="no_server", diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 9c4b4827..cb6a977f 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -129,10 +129,14 @@ class StatusBar(common.WWrap): r.append(":%s in file]"%self.master.server_playback.count()) else: r.append(":%s to go]"%self.master.server_playback.count()) - if self.master.get_ignore(): + if self.master.get_ignore_filter(): r.append("[") r.append(("heading_key", "I")) - r.append("gnore:%d]"%len(self.master.get_ignore())) + r.append("gnore:%d]" % len(self.master.get_ignore_filter())) + if self.master.get_tcp_filter(): + r.append("[") + r.append(("heading_key", "T")) + r.append("CP:%d]" % len(self.master.get_tcp_filter())) if self.master.state.intercept_txt: r.append("[") r.append(("heading_key", "i")) @@ -798,9 +802,13 @@ class ConsoleMaster(flow.FlowMaster): for command in commands: self.load_script(command) - def edit_ignore(self, ignore): + def edit_ignore_filter(self, ignore): patterns = (x[0] for x in ignore) - self.set_ignore(patterns) + self.set_ignore_filter(patterns) + + def edit_tcp_filter(self, tcp): + patterns = (x[0] for x in tcp) + self.set_tcp_filter(patterns) def loop(self): changed = True @@ -860,10 +868,18 @@ class ConsoleMaster(flow.FlowMaster): ) elif k == "I": self.view_grideditor( - grideditor.IgnoreEditor( + grideditor.HostPatternEditor( + self, + [[x] for x in self.get_ignore_filter()], + self.edit_ignore_filter + ) + ) + elif k == "T": + self.view_grideditor( + grideditor.HostPatternEditor( self, - [[x] for x in self.get_ignore()], - self.edit_ignore + [[x] for x in self.get_tcp_filter()], + self.edit_tcp_filter ) ) elif k == "i": diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index d629ec82..1673d536 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -495,8 +495,8 @@ class ScriptEditor(GridEditor): return str(v) -class IgnoreEditor(GridEditor): - title = "Editing ignore patterns" +class HostPatternEditor(GridEditor): + title = "Editing host patterns" columns = 1 headings = ("Regex (matched on hostname:port / ip:port)",) diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index bdcf3fd9..27288a36 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -119,6 +119,7 @@ class HelpView(urwid.ListBox): ("s", "add/remove scripts"), ("S", "server replay"), ("t", "set sticky cookie expression"), + ("T", "set tcp proxying pattern"), ("u", "set sticky auth expression"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 440798bc..5c3a0c7e 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,7 +11,7 @@ import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle -from .proxy.config import parse_host_pattern +from .proxy.config import HostMatcher import urlparse ODict = odict.ODict @@ -515,11 +515,17 @@ class FlowMaster(controller.Master): for script in self.scripts: self.run_single_script_hook(script, name, *args, **kwargs) - def get_ignore(self): - return [i.pattern for i in self.server.config.ignore] + def get_ignore_filter(self): + return self.server.config.check_ignore.patterns - def set_ignore(self, ignore): - self.server.config.ignore = parse_host_pattern(ignore) + def set_ignore_filter(self, host_patterns): + self.server.config.check_ignore = HostMatcher(host_patterns) + + def get_tcp_filter(self): + return self.server.config.check_tcp.patterns + + def set_tcp_filter(self, host_patterns): + self.server.config.check_tcp = HostMatcher(host_patterns) def set_stickycookie(self, txt): if txt: @@ -787,7 +793,7 @@ class FlowReader: v = ".".join(str(i) for i in data["version"]) raise FlowReadError("Incompatible serialized data version: %s"%v) off = self.fo.tell() - yield handle.protocols[data["conntype"]]["flow"].from_state(data) + yield handle.protocols[data["type"]]["flow"].from_state(data) except ValueError, v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 32a88b4b..33d860ca 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1260,9 +1260,9 @@ class HTTPHandler(ProtocolHandler): Returns False, if the connection should be closed immediately. """ address = tcp.Address.wrap(address) - if self.c.check_ignore_address(address): + if self.c.config.check_ignore(address): self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c).handle_messages() + TCPHandler(self.c, log=False).handle_messages() return False else: self.expected_form_in = "relative" @@ -1274,6 +1274,11 @@ class HTTPHandler(ProtocolHandler): self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.", "debug") + if self.c.config.check_tcp(address): + self.c.log("Generic TCP mode for host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + return True def authenticate(self, request): diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 519693db..1bf7f832 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -59,8 +59,8 @@ class Flow(stateobject.StateObject): A Flow is a collection of objects representing a single transaction. This class is usually subclassed for each protocol, e.g. HTTPFlow. """ - def __init__(self, conntype, client_conn, server_conn, live=None): - self.conntype = conntype + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type self.id = str(uuid.uuid4()) self.client_conn = client_conn """@type: ClientConnection""" @@ -78,7 +78,7 @@ class Flow(stateobject.StateObject): error=Error, client_conn=ClientConnection, server_conn=ServerConnection, - conntype=str + type=str ) def get_state(self, short=False): diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index a56bf07b..da0c9087 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler): chunk_size = 4096 + def __init__(self, c, log=True): + super(TCPHandler, self).__init__(c) + self.log = log + def handle_messages(self): self.c.establish_server_connection() @@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send # bytes/strings if not src.ssl_established: - # only ssl to dst, i.e. we revc'd into buf but need - # bytes/string now. + # we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(contents) + ), + "info" + ) dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - self.c.log( - "%s %s\r\n%s" % ( - direction, - dst_str, - cleanBin(buf.tobytes()) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(buf.tobytes()) + ), + "info" + ) dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index abdb7c41..948decc1 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import os import re -from netlib import http_auth, certutils +from netlib import http_auth, certutils, tcp from .. import utils, platform, version from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode @@ -10,8 +10,21 @@ CONF_BASENAME = "mitmproxy" CONF_DIR = "~/.mitmproxy" -def parse_host_pattern(patterns): - return [re.compile(p, re.IGNORECASE) for p in patterns] +class HostMatcher(object): + def __init__(self, patterns=[]): + self.patterns = list(patterns) + self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] + + def __call__(self, address): + address = tcp.Address.wrap(address) + host = "%s:%s" % (address.host, address.port) + if any(rex.search(host) for rex in self.regexes): + return True + else: + return False + + def __nonzero__(self): + return bool(self.patterns) class ProxyConfig: @@ -19,7 +32,7 @@ class ProxyConfig: confdir=CONF_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, - authenticator=None, ignore=[], + authenticator=None, ignore_hosts=[], tcp_hosts=[], ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS): self.host = host self.port = port @@ -44,7 +57,8 @@ class ProxyConfig: self.mode.http_form_in = http_form_in or self.mode.http_form_in self.mode.http_form_out = http_form_out or self.mode.http_form_out - self.ignore = parse_host_pattern(ignore) + self.check_ignore = HostMatcher(ignore_hosts) + self.check_tcp = HostMatcher(tcp_hosts) self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) @@ -124,7 +138,8 @@ def process_proxy_options(parser, options): upstream_server=upstream_server, http_form_in=options.http_form_in, http_form_out=options.http_form_out, - ignore=options.ignore, + ignore_hosts=options.ignore_hosts, + tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers=options.ciphers, certs=certs, diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 4c7fbbf0..fdf6405a 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -70,13 +70,15 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? client_ssl, server_ssl = False, False + conn_kwargs = dict() upstream_info = self.config.mode.get_upstream_server(self.client_conn) if upstream_info: self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] - if self.check_ignore_address(self.server_conn.address): + if self.config.check_ignore(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") self.conntype = "tcp" + conn_kwargs["log"] = False client_ssl, server_ssl = False, False else: pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form) @@ -90,15 +92,19 @@ class ConnectionHandler: if client_ssl or server_ssl: self.establish_ssl(client=client_ssl, server=server_ssl) + if self.config.check_tcp(self.server_conn.address): + self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info") + self.conntype = "tcp" + # Delegate handling to the protocol handler - protocol_handler(self.conntype)(self).handle_messages() + protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages() self.del_server_connection() self.log("clientdisconnect", "info") self.channel.tell("clientdisconnect", self) except ProxyError as e: - protocol_handler(self.conntype)(self).handle_error(e) + protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) except Exception: import traceback, sys @@ -119,14 +125,6 @@ class ConnectionHandler: self.server_conn = None self.sni = None - def check_ignore_address(self, address): - address = tcp.Address.wrap(address) - host = "%s:%s" % (address.host, address.port) - if host and any(rex.search(host) for rex in self.config.ignore): - return True - else: - return False - def set_server_address(self, address): """ Sets a new server address with the given priority. diff --git a/libmproxy/web/static/flows.json b/libmproxy/web/static/flows.json index a0358db0..35accd38 100644 --- a/libmproxy/web/static/flows.json +++ b/libmproxy/web/static/flows.json @@ -93,7 +93,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -259,7 +259,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -425,7 +425,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -595,7 +595,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -765,7 +765,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -919,7 +919,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1057,7 +1057,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1195,7 +1195,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1329,7 +1329,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1483,7 +1483,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1633,7 +1633,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1767,7 +1767,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1901,7 +1901,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -2027,7 +2027,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 -- cgit v1.2.3 From 37cc6ae0bbb32e528435f821469d36055574a810 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 19 Oct 2014 01:26:08 +0200 Subject: fix race conditions in tests --- libmproxy/flow.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 5c3a0c7e..d313c94a 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -662,6 +662,8 @@ class FlowMaster(controller.Master): """ Returns None if successful, or error message if not. """ + if f.live: + return "Can't replay request which is still live..." if f.intercepting: return "Can't replay while intercepting..." if f.request.content == http.CONTENT_MISSING: -- cgit v1.2.3 From 6cef6fbfec92f1154b6a5b986548478137598975 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 21 Oct 2014 15:08:39 +0200 Subject: tweak SSL detection heuristics --- libmproxy/protocol/http.py | 10 +++++++++- libmproxy/proxy/config.py | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 33d860ca..adb743a2 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1269,7 +1269,15 @@ class HTTPHandler(ProtocolHandler): self.expected_form_out = "relative" self.skip_authentication = True - if address.port in self.c.config.ssl_ports: + # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards. + # If we don't delegate to TCP mode, we should always negotiate a SSL connection. + should_establish_ssl = ( + address.port in self.c.config.ssl_ports + or + not self.c.config.check_tcp(address) + ) + + if should_establish_ssl: self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug") self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.", "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 948decc1..fe2b45f4 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -127,6 +127,12 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) + ssl_ports = options.ssl_ports + if options.ssl_ports != TRANSPARENT_SSL_PORTS: + # arparse appends to default value by default, strip that off. + # see http://bugs.python.org/issue16399 + ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] + return ProxyConfig( host=options.addr, port=options.port, @@ -144,6 +150,7 @@ def process_proxy_options(parser, options): ciphers=options.ciphers, certs=certs, certforward=options.certforward, + ssl_ports=ssl_ports ) @@ -180,7 +187,7 @@ def ssl_option_group(parser): help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-port", action="append", type=int, dest="ssl_ports", default=TRANSPARENT_SSL_PORTS, + "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), metavar="PORT", help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) -- cgit v1.2.3 From 1ef74cf294dd0fc1d2555e5256e1b1d39ca5fec5 Mon Sep 17 00:00:00 2001 From: Wade 524 Date: Fri, 24 Oct 2014 15:54:51 -0700 Subject: Fixing issue #368. --- libmproxy/flow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index e5fdf424..13895a05 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -219,9 +219,10 @@ class ServerPlaybackState: queriesArray = urlparse.parse_qsl(query) filtered = [] + ignore_params = self.ignore_params or [] for p in queriesArray: - if p[0] not in self.ignore_params: - filtered.append(p) + if p[0] not in ignore_params: + filtered.append(p) key = [ str(r.host), -- cgit v1.2.3 From efd6fdb0e24532de757fc90a8d3ae984b7170c51 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 17:13:25 +1300 Subject: Start a fuzzing architecture for mitmproxy --- libmproxy/dump.py | 11 +++++++---- libmproxy/proxy/server.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/dump.py b/libmproxy/dump.py index ccb2b5b5..9fb0f001 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -1,10 +1,13 @@ from __future__ import absolute_import -import sys, os +import sys +import os import netlib.utils from . import flow, filt, utils from .protocol import http -class DumpError(Exception): pass + +class DumpError(Exception): + pass class Options(object): @@ -37,6 +40,7 @@ class Options(object): "replay_ignore_content", "replay_ignore_params", ] + def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) @@ -71,7 +75,7 @@ class DumpMaster(flow.FlowMaster): self.anticache = options.anticache self.anticomp = options.anticomp self.showhost = options.showhost - self.replay_ignore_params = options.replay_ignore_params + self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.refresh_server_playback = options.refresh_server_playback @@ -88,7 +92,6 @@ class DumpMaster(flow.FlowMaster): if options.stickyauth: self.set_stickyauth(options.stickyauth) - if options.wfile: path = os.path.expanduser(options.wfile) try: diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index fdf6405a..613662c3 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -275,4 +275,4 @@ class ConnectionHandler: # make dang sure it doesn't happen. except: # pragma: no cover import traceback - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") \ No newline at end of file + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") -- cgit v1.2.3 From 7aee9a7c311e755147b398b8ba0b44aaec40eaf7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 17:44:49 +1300 Subject: Spacing and legibility --- libmproxy/flow.py | 9 +++-- libmproxy/protocol/http.py | 90 +++++++++++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 27 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 13895a05..6a24cc63 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -466,7 +466,7 @@ class FlowMaster(controller.Master): self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() - self.replay_ignore_params = False + self.replay_ignore_params = False self.replay_ignore_content = None @@ -719,7 +719,11 @@ class FlowMaster(controller.Master): if f.live: app = self.apps.get(f.request) if app: - err = app.serve(f, f.client_conn.wfile, **{"mitmproxy.master": self}) + err = app.serve( + f, + f.client_conn.wfile, + **{"mitmproxy.master": self} + ) if err: self.add_event("Error in wsgi app. %s"%err, "error") f.reply(protocol.KILL) @@ -769,7 +773,6 @@ class FlowMaster(controller.Master): self.stream = None - class FlowWriter: def __init__(self, fo): self.fo = fo diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index adb743a2..9542cc81 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -313,25 +313,37 @@ class HTTPRequest(HTTPMessage): request_line_parts = http.parse_init(request_line) if not request_line_parts: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) method, path, httpversion = request_line_parts if path == '*' or path.startswith("/"): form_in = "relative" if not netlib.utils.isascii(path): - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) @@ -343,23 +355,39 @@ class HTTPRequest(HTTPMessage): method, None, True) timestamp_end = utils.timestamp() - return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start, timestamp_end) + return HTTPRequest( + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) def _assemble_first_line(self, form=None): form = form or self.form_out if form == "relative": path = self.path if self.method != "OPTIONS" else "*" - request_line = '%s %s HTTP/%s.%s' % \ - (self.method, path, self.httpversion[0], self.httpversion[1]) + request_line = '%s %s HTTP/%s.%s' % ( + self.method, path, self.httpversion[0], self.httpversion[1] + ) elif form == "authority": - request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s:%s HTTP/%s.%s' % ( + self.method, self.host, self.port, self.httpversion[0], + self.httpversion[1] + ) elif form == "absolute": - request_line = '%s %s://%s:%s%s HTTP/%s.%s' % \ - (self.method, self.scheme, self.host, self.port, self.path, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s://%s:%s%s HTTP/%s.%s' % ( + self.method, self.scheme, self.host, + self.port, self.path, self.httpversion[0], + self.httpversion[1] + ) else: raise http.HttpError(400, "Invalid request form") return request_line @@ -371,7 +399,8 @@ class HTTPRequest(HTTPMessage): 'Connection', 'Transfer-Encoding']: del headers[k] - if headers["Upgrade"] == ["h2c"]: # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http + if headers["Upgrade"] == ["h2c"]: + # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http del headers["Upgrade"] if not 'host' in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, @@ -380,13 +409,16 @@ class HTTPRequest(HTTPMessage): if self.content: headers["Content-Length"] = [str(len(self.content))] - elif 'Transfer-Encoding' in self.headers: # content-length for e.g. chuncked transfer-encoding with no content + elif 'Transfer-Encoding' in self.headers: + # content-length for e.g. chuncked transfer-encoding with no content headers["Content-Length"] = ["0"] return str(headers) def _assemble_head(self, form=None): - return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers()) + return "%s\r\n%s\r\n" % ( + self._assemble_first_line(form), self._assemble_headers() + ) def assemble(self, form=None): """ @@ -396,7 +428,10 @@ class HTTPRequest(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head(form) if self.content: return head + self.content @@ -937,16 +972,23 @@ class HTTPHandler(ProtocolHandler): try: self.c.server_conn.send(request_raw) # Only get the headers at first... - flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False) + flow.response = HTTPResponse.from_stream( + self.c.server_conn.rfile, flow.request.method, + body_size_limit=self.c.config.body_size_limit, + include_body=False + ) break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: - self.c.log("error in server communication: %s" % repr(v), level="debug") + self.c.log( + "error in server communication: %s" % repr(v), + level="debug" + ) if attempt == 0: - # In any case, we try to reconnect at least once. - # This is necessary because it might be possible that we already initiated an upstream connection - # after clientconnect that has already been expired, e.g consider the following event log: + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: # > clientconnect (transparent mode destination known) # > serverconnect # > read n% of large request -- cgit v1.2.3 From 16654ad6a4ba4f12287d5707dafe3794b6e33fb8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 17:58:36 +1300 Subject: Fix crash while streaming Found using fuzzing. Reproduction with pathoc, given "mitmproxy -s" and pathod running on 9999: get:'http://localhost:9999/p/':s'200:b\'foo\':h\'Content-Length\'=\'3\'':i58,'\x1a':r return flow.FlowMaster.run(self) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 111, in run self.tick(self.masterq, 0.01) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 613, in tick return controller.Master.tick(self, q, timeout) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 101, in tick self.handle(*msg) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 118, in handle m(obj) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 738, in handle_responseheaders self.stream_large_bodies.run(f, False) File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 155, in run r.headers, is_request, flow.request.method, code File "/Users/aldo/mitmproxy/mitmproxy/netlib/http.py", line 401, in expected_http_body_size raise HttpError(400 if is_request else 502, "Invalid content-length header: %s" % headers["content-length"]) netlib.http.HttpError: Invalid content-length header: ['\x1a3'] --- libmproxy/cmdline.py | 27 +++++++++++++++++---------- libmproxy/flow.py | 8 ++++++-- libmproxy/protocol/http.py | 40 +++++++++++++++++++++++++++------------- 3 files changed, 50 insertions(+), 25 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 83eab7ee..4a3b5a48 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -248,14 +248,18 @@ def common_options(parser): "--stream", action="store", dest="stream_large_bodies", default=None, metavar="SIZE", - help="Stream data to the client if response body exceeds the given threshold. " - "If streamed, the body will not be stored in any way. Understands k/m/g suffixes, i.e. 3m for 3 megabytes." + help=""" + Stream data to the client if response body exceeds the given threshold. + If streamed, the body will not be stored in any way. Understands k/m/g + suffixes, i.e. 3m for 3 megabytes. + """ ) group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't do that because - # - --upstream-server should be in that group as well, but it's already in a different group. - # - our own error messages are more helpful + # We could make a mutually exclusive group out of -R, -U, -T, but we don't + # do that because - --upstream-server should be in that group as well, but + # it's already in a different group. - our own error messages are more + # helpful group.add_argument( "-b", action="store", type=str, dest="addr", default='', @@ -265,11 +269,14 @@ def common_options(parser): "-I", "--ignore", action="append", type=str, dest="ignore_hosts", default=[], metavar="HOST", - help="Ignore host and forward all traffic without processing it. " - "In transparent mode, it is recommended to use an IP address (range), not the hostname. " - "In regular mode, only SSL traffic is ignored and the hostname should be used. " - "The supplied value is interpreted as a regular expression and matched on the ip or the hostname. " - "Can be passed multiple times. " + help=""" + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. Can be + passed multiple times. + """ ) group.add_argument( "--tcp", diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 6a24cc63..6136ec1c 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -738,8 +738,12 @@ class FlowMaster(controller.Master): def handle_responseheaders(self, f): self.run_script_hook("responseheaders", f) - if self.stream_large_bodies: - self.stream_large_bodies.run(f, False) + try: + if self.stream_large_bodies: + self.stream_large_bodies.run(f, False) + except netlib.http.HttpError: + f.reply(protocol.KILL) + return f.reply() return f diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 9542cc81..e81c7640 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -18,6 +18,10 @@ HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" CONTENT_MISSING = 0 +class KillSignal(Exception): + pass + + def get_line(fp): """ Get a line, possibly preceded by a blank. @@ -1001,19 +1005,21 @@ class HTTPHandler(ProtocolHandler): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow) - - # now get the rest of the request body, if body still needs to be read - # but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING + flow = self.c.channel.ask("responseheaders", flow) + if flow == KILL: + raise KillSignal else: - flow.response.content = http.read_http_body( - self.c.server_conn.rfile, flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, flow.response.code, False - ) - flow.response.timestamp_end = utils.timestamp() + # now get the rest of the request body, if body still needs to be + # read but not streaming this response + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = http.read_http_body( + self.c.server_conn.rfile, flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, flow.response.code, False + ) + flow.response.timestamp_end = utils.timestamp() def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) @@ -1092,8 +1098,16 @@ class HTTPHandler(ProtocolHandler): flow.live.restore_server() return True # Next flow please. - except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: + except ( + HttpAuthenticationError, + http.HttpError, + proxy.ProxyError, + tcp.NetLibError, + ), e: self.handle_error(e, flow) + except KillSignal: + self.c.log("Connection killed", "info") + flow.live = None finally: flow.live = None # Connection is not live anymore. return False -- cgit v1.2.3 From 340d0570bfe7ceae68d7d592e3b7283480c351b0 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 18:32:45 +1300 Subject: Legibility --- libmproxy/protocol/http.py | 172 ++++++++++++++++++++++++++++++++------------- 1 file changed, 124 insertions(+), 48 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index e81c7640..3560f0bd 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -27,7 +27,8 @@ def get_line(fp): Get a line, possibly preceded by a blank. """ line = fp.readline() - if line == "\r\n" or line == "\n": # Possible leftover from previous message + if line == "\r\n" or line == "\n": + # Possible leftover from previous message line = fp.readline() if line == "": raise tcp.NetLibDisconnect() @@ -241,25 +242,47 @@ class HTTPRequest(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - form_in: The request form which mitmproxy has received. The following values are possible: - - relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + form_in: The request form which mitmproxy has received. The following + values are possible: - form_out: The request form which mitmproxy has send out to the destination + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + + form_out: The request form which mitmproxy has send out to the + destination timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended """ - def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start=None, timestamp_end=None, form_out=None): + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start=None, + timestamp_end=None, + form_out=None + ): assert isinstance(headers, ODictCaseless) or not headers - HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start, - timestamp_end) - + HTTPMessage.__init__( + self, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) self.form_in = form_in self.method = method self.scheme = scheme @@ -312,7 +335,8 @@ class HTTPRequest(HTTPMessage): request_line = get_line(rfile) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp request_line_parts = http.parse_init(request_line) @@ -683,7 +707,9 @@ class HTTPResponse(HTTPMessage): return "".format( code=self.code, msg=self.msg, - contenttype=self.headers.get_first("content-type", "unknown content type"), + contenttype=self.headers.get_first( + "content-type", "unknown content type" + ), size=size ) @@ -704,7 +730,8 @@ class HTTPResponse(HTTPMessage): body_size_limit, include_body=include_body) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: @@ -745,7 +772,11 @@ class HTTPResponse(HTTPMessage): def _assemble_head(self, preserve_transfer_encoding=False): return '%s\r\n%s\r\n' % ( - self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) + self._assemble_first_line(), + self._assemble_headers( + preserve_transfer_encoding=preserve_transfer_encoding + ) + ) def assemble(self): """ @@ -755,7 +786,10 @@ class HTTPResponse(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head() if self.content: return head + self.content @@ -822,8 +856,9 @@ class HTTPResponse(HTTPMessage): pairs = [pair.partition("=") for pair in header.split(';')] cookie_name = pairs[0][0] # the key of the first key/value pairs cookie_value = pairs[0][2] # the value of the first key/value pairs - cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in - pairs[1:]} + cookie_parameters = { + key.strip().lower(): value.strip() for key, sep, value in pairs[1:] + } cookies.append((cookie_name, (cookie_value, cookie_parameters))) return dict(cookies) @@ -856,7 +891,8 @@ class HTTPFlow(Flow): self.response = None """@type: HTTPResponse""" - self.intercepting = False # FIXME: Should that rather be an attribute of Flow? + # FIXME: Should that rather be an attribute of Flow? + self.intercepting = False _stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes.update( @@ -944,7 +980,9 @@ class HTTPFlow(Flow): class HttpAuthenticationError(Exception): def __init__(self, auth_headers=None): - super(HttpAuthenticationError, self).__init__("Proxy Authentication Required") + super(HttpAuthenticationError, self).__init__( + "Proxy Authentication Required" + ) self.headers = auth_headers self.code = 407 @@ -1114,7 +1152,12 @@ class HTTPHandler(ProtocolHandler): def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) + send_connect_request( + self.c.server_conn, + state["host"], + state["port"], + update_state=False + ) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1138,11 +1181,11 @@ class HTTPHandler(ProtocolHandler): self.c.log(message_debug, level="debug") if flow: - # TODO: no flows without request or with both request and response at the moment. + # TODO: no flows without request or with both request and response + # at the moment. if flow.request and not flow.response: flow.error = Error(message or message_debug) self.c.channel.ask("error", flow) - try: code = getattr(error, "code", 502) headers = getattr(error, "headers", None) @@ -1156,12 +1199,22 @@ class HTTPHandler(ProtocolHandler): def send_error(self, code, message, headers): response = http_status.RESPONSES.get(code, "Unknown") - html_content = '\n%d %s\n\n\n%s\n\n' % \ - (code, response, message) + html_content = """ + + + %d %s + + + + """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) - self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version) + self.c.client_conn.wfile.write( + "Server: %s\r\n" % self.c.config.server_version + ) self.c.client_conn.wfile.write("Content-type: text/html\r\n") - self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) + self.c.client_conn.wfile.write( + "Content-Length: %d\r\n" % len(html_content) + ) if headers: for key, value in headers.items(): self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) @@ -1201,11 +1254,15 @@ class HTTPHandler(ProtocolHandler): # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: - raise http.HttpError(400, "Must not CONNECT on already encrypted connection") + raise http.HttpError( + 400, + "Must not CONNECT on already encrypted connection" + ) if self.c.config.mode == "regular": self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + # Update server_conn attribute on the flow + flow.server_conn = self.c.server_conn self.c.establish_server_connection() self.c.client_conn.send( 'HTTP/1.1 200 Connection established\r\n' + @@ -1217,7 +1274,9 @@ class HTTPHandler(ProtocolHandler): elif self.c.config.mode == "upstream": return None else: - pass # CONNECT should never occur if we don't expect absolute-form requests + # CONNECT should never occur if we don't expect absolute-form + # requests + pass elif request.form_in == self.expected_form_in: @@ -1225,61 +1284,77 @@ class HTTPHandler(ProtocolHandler): if request.form_in == "absolute": if request.scheme != "http": - raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + raise http.HttpError( + 400, + "Invalid request scheme: %s" % request.scheme + ) if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct value at flow.server_conn + # Update info so that an inline script sees the correct + # value at flow.server_conn self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn return None - - raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % - (self.expected_form_in, request.form_in)) + raise http.HttpError( + 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( + self.expected_form_in, request.form_in + ) + ) def process_server_address(self, flow): # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be unfiddled here + # We provide a mostly unified API to the user, which needs to be + # unfiddled here # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) address = netlib.tcp.Address((flow.request.host, flow.request.port)) ssl = (flow.request.scheme == "https") if self.c.config.mode == "upstream": - - # The connection to the upstream proxy may have a state we may need to take into account. + # The connection to the upstream proxy may have a state we may need + # to take into account. connected_to = None for s in flow.server_conn.state: if s[0] == "http" and s[1]["state"] == "connect": connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - # We need to reconnect if the current flow either requires a (possibly impossible) - # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + # We need to reconnect if the current flow either requires a + # (possibly impossible) change to the connection state, e.g. the + # host has changed but we already CONNECTed somewhere else. needs_server_change = ( ssl != self.c.server_conn.ssl_established or - (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + # HTTP proxying is "stateless", CONNECT isn't. + (connected_to and address != connected_to) ) if needs_server_change: # force create new connection to the proxy server to reset state self.live.change_server(self.c.server_conn.address, force=True) if ssl: - send_connect_request(self.c.server_conn, address.host, address.port) + send_connect_request( + self.c.server_conn, + address.host, + address.port + ) self.c.establish_ssl(server=True) else: - # If we're not in upstream mode, we just want to update the host and possibly establish TLS. - self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + # If we're not in upstream mode, we just want to update the host and + # possibly establish TLS. This is a no op if the addresses match. + self.live.change_server(address, ssl=ssl) flow.server_conn = self.c.server_conn def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: - # we already received the full response from the server and can send it to the client straight away. + # we already received the full response from the server and can send + # it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: - # First send the headers and then transfer the response incrementally: + # First send the headers and then transfer the response + # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, @@ -1293,7 +1368,8 @@ class HTTPHandler(ProtocolHandler): def check_close_connection(self, flow): """ - Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. """ close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or -- cgit v1.2.3 From 3b0964f36555949d35659f306054876a49dbcfa1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 30 Oct 2014 17:38:23 +0100 Subject: fix #391 --- libmproxy/proxy/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index fe2b45f4..a228192a 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -43,9 +43,9 @@ class ProxyConfig: self.body_size_limit = body_size_limit if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), TRANSPARENT_SSL_PORTS) + self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) elif mode == "socks5": - self.mode = Socks5ProxyMode(TRANSPARENT_SSL_PORTS) + self.mode = Socks5ProxyMode(ssl_ports) elif mode == "reverse": self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": -- cgit v1.2.3 From ce18cd8ba40998c0654e697efcc0a0f018e45375 Mon Sep 17 00:00:00 2001 From: Wade 524 Date: Fri, 31 Oct 2014 11:50:03 -0700 Subject: Fixing issue #392. --- libmproxy/protocol/http.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3560f0bd..1472f2ca 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -251,7 +251,7 @@ class HTTPRequest(HTTPMessage): - authority-form (CONNECT example.com:443) Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - form_out: The request form which mitmproxy has send out to the + form_out: The request form which mitmproxy will send out to the destination timestamp_start: Timestamp indicating when request transmission started @@ -401,9 +401,8 @@ class HTTPRequest(HTTPMessage): form = form or self.form_out if form == "relative": - path = self.path if self.method != "OPTIONS" else "*" request_line = '%s %s HTTP/%s.%s' % ( - self.method, path, self.httpversion[0], self.httpversion[1] + self.method, self.path, self.httpversion[0], self.httpversion[1] ) elif form == "authority": request_line = '%s %s:%s HTTP/%s.%s' % ( -- cgit v1.2.3 From 0fe83ce87bd1e3cf00fb06d2cc06e1bf3b0dbe85 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 6 Nov 2014 10:35:00 +1300 Subject: Fix bug in flow dumping, add unit test that should have caught this in the first place --- libmproxy/dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 9fb0f001..4d899fe8 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -205,7 +205,7 @@ class DumpMaster(flow.FlowMaster): elif self.o.flow_detail >= 3: print >> self.outfile, str_request(f, self.showhost) print >> self.outfile, self.indent(4, f.request.headers) - if utils.isBin(f.request.content): + if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content): d = netlib.utils.hexdump(f.request.content) d = "\n".join("%s\t%s %s"%i for i in d) print >> self.outfile, self.indent(4, d) -- cgit v1.2.3 From a2a87695d32df6bcf31a0785496d8ca6e41423c8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 6 Nov 2014 10:51:30 +1300 Subject: Reduce loop timeouts to improve mitmproxy responsiveness Fixes #384 --- libmproxy/console/__init__.py | 2 +- libmproxy/flow.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index cb6a977f..ffd9eda8 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -819,7 +819,7 @@ class ConsoleMaster(flow.FlowMaster): self.statusbar.redraw() size = self.drawscreen() changed = self.tick(self.masterq, 0.01) - self.ui.set_input_timeouts(max_wait=0.1) + self.ui.set_input_timeouts(max_wait=0.01) keys = self.ui.get_input() if keys: changed = True diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 6136ec1c..bd35e864 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -27,7 +27,12 @@ class AppRegistry: Add a WSGI app to the registry, to be served for requests to the specified domain, on the specified port. """ - self.apps[(domain, port)] = wsgi.WSGIAdaptor(app, domain, port, version.NAMEVERSION) + self.apps[(domain, port)] = wsgi.WSGIAdaptor( + app, + domain, + port, + version.NAMEVERSION + ) def get(self, request): """ @@ -72,7 +77,8 @@ class ReplaceHooks: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] @@ -119,7 +125,8 @@ class SetHeaders: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] -- cgit v1.2.3 From dc142682cb930cb3903a2fc66d4785bd5367360b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Nov 2014 11:25:03 +0100 Subject: fix #399 --- libmproxy/console/__init__.py | 3 ++- libmproxy/console/flowlist.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index ffd9eda8..fc6600c1 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -516,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster): self.start_server_playback( ret, self.killextra, self.rheaders, - False, self.nopop + False, self.nopop, + self.options.replay_ignore_params, self.options.replay_ignore_content ) def spawn_editor(self, data): diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index e0330171..3eb4eb1a 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -120,13 +120,15 @@ class ConnectionItem(common.WWrap): self.master.start_server_playback( [i.copy() for i in self.master.state.view], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) elif k == "t": self.master.start_server_playback( [self.flow.copy()], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) else: self.master.path_prompt( -- cgit v1.2.3 From c3ec5515466fe7ba970bc7cb2578dad4c845e5bc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 7 Nov 2014 09:52:46 +0100 Subject: fix #401 --- libmproxy/protocol/http.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 1472f2ca..c8974d25 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -434,11 +434,9 @@ class HTTPRequest(HTTPMessage): self.host, self.port)] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - elif 'Transfer-Encoding' in self.headers: - # content-length for e.g. chuncked transfer-encoding with no content - headers["Content-Length"] = ["0"] return str(headers) @@ -761,11 +759,9 @@ class HTTPResponse(HTTPMessage): if not preserve_transfer_encoding: del headers['Transfer-Encoding'] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - # add content-length for chuncked transfer-encoding with no content - elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: - headers["Content-Length"] = ["0"] return str(headers) -- cgit v1.2.3 From 6f5883a4d19caee30a89729ef5f0f4e50da8359c Mon Sep 17 00:00:00 2001 From: Lucas Cimon Date: Fri, 7 Nov 2014 17:01:58 +0100 Subject: Using uppercase C to 'clear' display mode, because lowercase 'c' is used for css --- libmproxy/console/flowview.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index b2c46147..bf0070fc 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -637,11 +637,10 @@ class FlowView(common.WWrap): return self._view_nextprev_flow("prev", flow) def change_this_display_mode(self, t): - self.state.add_flow_setting( - self.flow, - (self.state.view_flow_mode, "prettyview"), - contentview.get_by_shortcut(t) - ) + key = (self.state.view_flow_mode, "prettyview") + value = contentview.get_by_shortcut(t) + if value: + self.state.add_flow_setting(self.flow, key, value) self.master.refresh_flow(self.flow) def delete_body(self, t): @@ -749,7 +748,7 @@ class FlowView(common.WWrap): self.master.statusbar.message("") elif key == "m": p = list(contentview.view_prompts) - p.insert(0, ("clear", "c")) + p.insert(0, ("Clear", "C")) self.master.prompt_onekey( "Display mode", p, -- cgit v1.2.3 From cece3700df02144dc2bd26a55fe4d6076fd7ea22 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 10 Nov 2014 17:11:36 +0100 Subject: fix #402 --- libmproxy/flow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index bd35e864..1826af3d 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -192,6 +192,7 @@ class ClientPlaybackState: """ if self.flows and not self.current: n = self.flows.pop(0) + n.response = None n.reply = controller.DummyReply() self.current = master.handle_request(n) if not testing and not self.current.response: @@ -615,7 +616,7 @@ class FlowMaster(controller.Master): ] if all(e): self.shutdown() - self.client_playback.tick(self, timeout) + self.client_playback.tick(self) return controller.Master.tick(self, q, timeout) -- cgit v1.2.3 From 6f3b4eee3c3e30b391be457e38fb5ac67f8ef682 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 10 Nov 2014 17:35:28 +0100 Subject: fix clear key --- libmproxy/console/flowview.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index bf0070fc..3dceff70 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -637,10 +637,11 @@ class FlowView(common.WWrap): return self._view_nextprev_flow("prev", flow) def change_this_display_mode(self, t): - key = (self.state.view_flow_mode, "prettyview") - value = contentview.get_by_shortcut(t) - if value: - self.state.add_flow_setting(self.flow, key, value) + self.state.add_flow_setting( + self.flow, + (self.state.view_flow_mode, "prettyview"), + contentview.get_by_shortcut(t) + ) self.master.refresh_flow(self.flow) def delete_body(self, t): -- cgit v1.2.3 From f19ee74b99d0049f4789673e202cfc3fda59bb04 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 11 Nov 2014 12:30:51 +0100 Subject: be more explicit about requirements --- libmproxy/version.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/version.py b/libmproxy/version.py index 3483625d..f4c0cd94 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -3,3 +3,7 @@ VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" NAMEVERSION = NAME + " " + VERSION + +NEXT_MINORVERSION = list(IVERSION) +NEXT_MINORVERSION[1] += 1 +NEXT_MINORVERSION = ".".join(str(i) for i in NEXT_MINORVERSION[:2]) \ No newline at end of file -- cgit v1.2.3 From a325ae638b07a8a08018403e57c05ab8d4119161 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 11 Nov 2014 13:09:05 +0100 Subject: fix tests --- libmproxy/flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 1826af3d..3d5a6a36 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -191,7 +191,7 @@ class ClientPlaybackState: testing: Disables actual replay for testing. """ if self.flows and not self.current: - n = self.flows.pop(0) + n = self.flows.pop(0).copy() n.response = None n.reply = controller.DummyReply() self.current = master.handle_request(n) -- cgit v1.2.3 From 9b5a8af12d820fd3593424989759568e7cffd596 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Nov 2014 00:21:47 +0100 Subject: fix grideditor bug --- libmproxy/console/grideditor.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index 1673d536..72c1e4a0 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -129,6 +129,8 @@ class GridWalker(urwid.ListWalker): if emsg: self.editor.master.statusbar.message(emsg, 1000) errors.add(self.focus_col) + else: + errors.discard(self.focus_col) row = list(self.lst[self.focus][0]) row[self.focus_col] = val -- cgit v1.2.3 From 0c52b4e3b9137e22f6c8c649d6b584d44d7f4b75 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Nov 2014 00:26:22 +0100 Subject: handle script hooks in replay, fix tests, fix #402 --- libmproxy/flow.py | 24 ++++++++--------- libmproxy/protocol/http.py | 66 ++++++++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 40 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d5a6a36..00713698 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -169,6 +169,7 @@ class ClientPlaybackState: def __init__(self, flows, exit): self.flows, self.exit = flows, exit self.current = None + self.testing = False # Disables actual replay for testing. def count(self): return len(self.flows) @@ -186,19 +187,16 @@ class ClientPlaybackState: if flow is self.current: self.current = None - def tick(self, master, testing=False): - """ - testing: Disables actual replay for testing. - """ + def tick(self, master): if self.flows and not self.current: - n = self.flows.pop(0).copy() - n.response = None - n.reply = controller.DummyReply() - self.current = master.handle_request(n) - if not testing and not self.current.response: - master.replay_request(self.current) # pragma: no cover - elif self.current.response: - master.handle_response(self.current) + self.current = self.flows.pop(0).copy() + if not self.testing: + master.replay_request(self.current) + else: + self.current.reply = controller.DummyReply() + master.handle_request(self.current) + if self.current.response: + master.handle_response(self.current) class ServerPlaybackState: @@ -371,6 +369,8 @@ class State(object): """ Add a request to the state. Returns the matching flow. """ + if flow in self._flow_list: # catch flow replay + return flow self._flow_list.append(flow) if flow.match(self._limit): self.view.append(flow) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index c8974d25..26a94040 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1040,7 +1040,7 @@ class HTTPHandler(ProtocolHandler): # inline script to set flow.stream = True flow = self.c.channel.ask("responseheaders", flow) if flow == KILL: - raise KillSignal + raise KillSignal() else: # now get the rest of the request body, if body still needs to be # read but not streaming this response @@ -1085,7 +1085,7 @@ class HTTPHandler(ProtocolHandler): self.process_server_address(flow) # The inline script may have changed request.host if request_reply is None or request_reply == KILL: - return False + raise KillSignal() if isinstance(request_reply, HTTPResponse): flow.response = request_reply @@ -1099,7 +1099,7 @@ class HTTPHandler(ProtocolHandler): self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow) if response_reply is None or response_reply == KILL: - return False + raise KillSignal() self.send_response_to_client(flow) @@ -1140,7 +1140,6 @@ class HTTPHandler(ProtocolHandler): self.handle_error(e, flow) except KillSignal: self.c.log("Connection killed", "info") - flow.live = None finally: flow.live = None # Connection is not live anymore. return False @@ -1437,32 +1436,43 @@ class RequestReplayThread(threading.Thread): r = self.flow.request form_out_backup = r.form_out try: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) - r.form_out = "relative" - else: - r.form_out = "absolute" + self.flow.response = None + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise KillSignal() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) - r.form_out = "relative" - - server.send(r.assemble()) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, - body_size_limit=self.config.body_size_limit) - self.channel.ask("response", self.flow) - except (proxy.ProxyError, http.HttpError, tcp.NetLibError), v: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + + server.send(r.assemble()) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, + body_size_limit=self.config.body_size_limit) + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise KillSignal() + except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: self.flow.error = Error(repr(v)) self.channel.ask("error", self.flow) + except KillSignal: + self.channel.tell("log", proxy.Log("Connection killed", "info")) finally: r.form_out = form_out_backup -- cgit v1.2.3 From be449b7129a55af81e249f992046d88a02efbc46 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Nov 2014 16:13:45 +0100 Subject: fix #409 --- libmproxy/proxy/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 613662c3..55e2b30e 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -99,7 +99,6 @@ class ConnectionHandler: # Delegate handling to the protocol handler protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages() - self.del_server_connection() self.log("clientdisconnect", "info") self.channel.tell("clientdisconnect", self) @@ -112,6 +111,10 @@ class ConnectionHandler: print >> sys.stderr, traceback.format_exc() print >> sys.stderr, "mitmproxy has crashed!" print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" + finally: + # Make sure that we close the server connection in any case. + # The client connection is closed by the ProxyServer and does not have be handled here. + self.del_server_connection() def del_server_connection(self): """ @@ -119,6 +122,7 @@ class ConnectionHandler: """ if self.server_conn and self.server_conn.connection: self.server_conn.finish() + self.server_conn.close() self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host, self.server_conn.address.port)]) self.channel.tell("serverdisconnect", self) -- cgit v1.2.3 From afc6ef99eac9460fa2c6fb63d6a3ba528d69f443 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Nov 2014 16:18:05 +0100 Subject: bump version --- libmproxy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/version.py b/libmproxy/version.py index f4c0cd94..8dcaecc8 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -1,4 +1,4 @@ -IVERSION = (0, 11) +IVERSION = (0, 11, 1) VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" -- cgit v1.2.3 From c7a96b2fb121ba811aa5dbc580cb44fc6c4d2ed6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Nov 2014 00:52:26 +0100 Subject: always show error messages --- libmproxy/console/__init__.py | 2 +- libmproxy/dump.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index fc6600c1..e6bc9b41 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -1050,7 +1050,7 @@ class ConsoleMaster(flow.FlowMaster): self.eventlist[:] = [] def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.options.verbosity < needed: return diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 4d899fe8..0d9432c9 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -155,7 +155,7 @@ class DumpMaster(flow.FlowMaster): return flows def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.o.verbosity >= needed: print >> self.outfile, e self.outfile.flush() -- cgit v1.2.3 From 24c4df07e39d537a631c111df2eef36e8cb1bd70 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 16:14:08 +1300 Subject: First-order integration of configargparser to add config file support --- libmproxy/cmdline.py | 109 +++++++++++++++++++++++++++++------------- libmproxy/main.py | 77 ++++++++++++++++++++--------- libmproxy/onboarding/app.py | 4 +- libmproxy/platform/windows.py | 4 +- libmproxy/proxy/config.py | 35 ++++++++++---- 5 files changed, 160 insertions(+), 69 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 4a3b5a48..09e25ada 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import re -from argparse import ArgumentTypeError +from configargparse import ArgumentTypeError from netlib import http from . import filt, utils from .proxy import config @@ -22,7 +22,9 @@ def _parse_hook(s): elif len(parts) == 3: patt, a, b = parts else: - raise ParseException("Malformed hook specifier - too few clauses: %s" % s) + raise ParseException( + "Malformed hook specifier - too few clauses: %s" % s + ) if not a: raise ParseException("Empty clause: %s" % str(patt)) @@ -184,12 +186,16 @@ def common_options(parser): parser.add_argument( "--anticache", action="store_true", dest="anticache", default=False, - help="Strip out request headers that might cause the server to return 304-not-modified." + + help=""" + Strip out request headers that might cause the server to return + 304-not-modified. + """ ) parser.add_argument( - "--confdir", - action="store", type=str, dest="confdir", default='~/.mitmproxy', - help="Configuration directory, contains default CA file. (~/.mitmproxy)" + "--cadir", + action="store", type=str, dest="cadir", default=config.CA_DIR, + help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR ) parser.add_argument( "--host", @@ -210,11 +216,17 @@ def common_options(parser): "-s", action="append", type=str, dest="scripts", default=[], metavar='"script.py --bar"', - help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times." + help=""" + Run a script. Surround with quotes to pass script arguments. Can be + passed multiple times. + """ ) parser.add_argument( "-t", - action="store", dest="stickycookie_filt", default=None, metavar="FILTER", + action="store", + dest="stickycookie_filt", + default=None, + metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( @@ -241,7 +253,7 @@ def common_options(parser): "-Z", action="store", dest="body_size_limit", default=None, metavar="SIZE", - help="Byte size limit of HTTP request and response bodies." \ + help="Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) parser.add_argument( @@ -249,9 +261,9 @@ def common_options(parser): action="store", dest="stream_large_bodies", default=None, metavar="SIZE", help=""" - Stream data to the client if response body exceeds the given threshold. - If streamed, the body will not be stored in any way. Understands k/m/g - suffixes, i.e. 3m for 3 megabytes. + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. """ ) @@ -282,8 +294,11 @@ def common_options(parser): "--tcp", action="append", type=str, dest="tcp_hosts", default=[], metavar="HOST", - help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore," - "but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode." + help=""" + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the event log in verbose mode. + """ ) group.add_argument( "-n", @@ -297,8 +312,14 @@ def common_options(parser): ) group.add_argument( "-R", - action="store", type=parse_server_spec, dest="reverse_proxy", default=None, - help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]" + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ ) group.add_argument( "--socks", @@ -312,16 +333,20 @@ def common_options(parser): ) group.add_argument( "-U", - action="store", type=parse_server_spec, dest="upstream_proxy", default=None, + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) group = parser.add_argument_group( "Advanced Proxy Options", """ - The following options allow a custom adjustment of the proxy behavior. - Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T). - """.strip() + The following options allow a custom adjustment of the proxy + behavior. Normally, you don't want to use these options directly and + use the provided wrappers instead (-R, -U, -T). + """ ) group.add_argument( "--http-form-in", dest="http_form_in", default=None, @@ -343,13 +368,19 @@ def common_options(parser): group.add_argument( "--app-host", action="store", dest="app_host", default=APP_HOST, metavar="host", - help="Domain to serve the onboarding app from. For transparent mode, use an IP when\ - a DNS entry for the app domain is not present. Default: %s" % APP_HOST - + help=""" + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. Default: + %s + """ % APP_HOST ) group.add_argument( "--app-port", - action="store", dest="app_port", default=APP_PORT, type=int, metavar="80", + action="store", + dest="app_port", + default=APP_PORT, + type=int, + metavar="80", help="Port to serve the onboarding app from." ) @@ -380,8 +411,10 @@ def common_options(parser): group.add_argument( "--norefresh", action="store_true", dest="norefresh", default=False, - help="Disable response refresh, " - "which updates times in cookies and headers for replayed responses." + help=""" + Disable response refresh, which updates times in cookies and headers + for replayed responses. + """ ) group.add_argument( "--no-pop", @@ -392,13 +425,17 @@ def common_options(parser): group.add_argument( "--replay-ignore-content", action="store_true", dest="replay_ignore_content", default=False, - help="Ignore request's content while searching for a saved flow to replay" + help=""" + Ignore request's content while searching for a saved flow to replay + """ ) group.add_argument( "--replay-ignore-param", action="append", dest="replay_ignore_params", type=str, - help="Request's parameters to be ignored while searching for a saved flow to replay" - "Can be passed multiple times." + help=""" + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. + """ ) group = parser.add_argument_group( @@ -417,9 +454,12 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action="append", type=str, dest="replace_file", default=[], - metavar="PATH", - help="Replacement pattern, where the replacement clause is a path to a file." + action = "append", type=str, dest="replace_file", default=[], + metavar = "PATH", + help = """ + Replacement pattern, where the replacement clause is a path to a + file. + """ ) group = parser.add_argument_group( @@ -455,7 +495,10 @@ def common_options(parser): "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", - help="Allows access to a a single user, specified in the form username:password." + help=""" + Allows access to a a single user, specified in the form + username:password. + """ ) user_specification_group.add_argument( "--htpasswd", diff --git a/libmproxy/main.py b/libmproxy/main.py index 2d6a0119..9cad5dcc 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -1,4 +1,5 @@ from __future__ import print_function, absolute_import +import configargparse import argparse import os import signal @@ -11,25 +12,38 @@ from .proxy.server import DummyServer, ProxyServer def check_versions(): """ - Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error. - Check before every start that both versions are somewhat okay. + Having installed a wrong version of pyOpenSSL or netlib is unfortunately a + very common source of error. Check before every start that both versions are + somewhat okay. """ - # We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version. + # We don't introduce backward-incompatible changes in patch versions. Only + # consider major and minor version. if netlib.version.IVERSION[:2] != version.IVERSION[:2]: print( "Warning: You are using mitmdump %s with netlib %s. " - "Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION), - file=sys.stderr) - import OpenSSL, inspect - + "Most likely, that won't work - please upgrade!" % ( + version.VERSION, netlib.version.VERSION + ), + file=sys.stderr + ) + import OpenSSL + import inspect v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2]) if v < (0, 14): - print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.", - file=sys.stderr) - # Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got. + print( + "You are using an outdated version of pyOpenSSL:" + " mitmproxy requires pyOpenSSL 0.14 or greater.", + file=sys.stderr + ) + # Some users apparently have multiple versions of pyOpenSSL installed. + # Report which one we got. pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL)) - print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path), - file=sys.stderr) + print( + "Your pyOpenSSL %s installation is located at %s" % ( + OpenSSL.__version__, pyopenssl_path + ), + file=sys.stderr + ) sys.exit(1) @@ -38,8 +52,14 @@ def assert_utf8_env(): for i in ["LANG", "LC_CTYPE", "LC_ALL"]: spec += os.environ.get(i, "").lower() if "utf" not in spec: - print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr) - print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr) + print( + "Error: mitmproxy requires a UTF console environment.", + file=sys.stderr + ) + print( + "Set your LANG enviroment variable to something like en_US.UTF-8", + file=sys.stderr + ) sys.exit(1) @@ -55,12 +75,17 @@ def get_server(dummy_server, options): def mitmproxy_cmdline(): - # Don't import libmproxy.console for mitmdump, urwid is not available on all platforms. + # Don't import libmproxy.console for mitmdump, urwid is not available on all + # platforms. from . import console from .console import palettes - parser = argparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument('--version', action='version', version=version.NAMEVERSION) + parser = configargparse.ArgumentParser(usage="%(prog)s [options]") + parser.add_argument( + '--version', + action='version', + version=version.NAMEVERSION + ) cmdline.common_options(parser) parser.add_argument( "--palette", type=str, default="dark", @@ -113,13 +138,21 @@ def mitmproxy(): # pragma: nocover def mitmdump_cmdline(): from . import dump - parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") - parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION) + parser = configargparse.ArgumentParser(usage="%(prog)s [options] [filter]") + + parser.add_argument( + '--version', + action= 'version', + version= "mitmdump" + " " + version.VERSION + ) cmdline.common_options(parser) parser.add_argument( "--keepserving", - action="store_true", dest="keepserving", default=False, - help="Continue serving after client playback or file read. We exit by default." + action= "store_true", dest="keepserving", default=False, + help= """ + Continue serving after client playback or file read. We exit by + default. + """ ) parser.add_argument( "-d", @@ -166,7 +199,7 @@ def mitmdump(): # pragma: nocover def mitmweb_cmdline(): from . import web - parser = argparse.ArgumentParser(usage="%(prog)s [options]") + parser = configargparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument( '--version', action='version', diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py index 9b5db38a..4023fae2 100644 --- a/libmproxy/onboarding/app.py +++ b/libmproxy/onboarding/app.py @@ -18,12 +18,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/platform/windows.py b/libmproxy/platform/windows.py index ddbbed52..066a377d 100644 --- a/libmproxy/platform/windows.py +++ b/libmproxy/platform/windows.py @@ -1,4 +1,4 @@ -import argparse +import configargparse import cPickle as pickle from ctypes import byref, windll, Structure from ctypes.wintypes import DWORD @@ -361,7 +361,7 @@ class TransparentProxy(object): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Windows Transparent Proxy") + parser = configargparse.ArgumentParser(description="Windows Transparent Proxy") parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both", help='redirection operation mode: "forward" to only redirect forwarded packets, ' '"local" to only redirect packets originating from the local machine') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index a228192a..3d373a28 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -7,7 +7,7 @@ from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMod TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" -CONF_DIR = "~/.mitmproxy" +CA_DIR = "~/.mitmproxy" class HostMatcher(object): @@ -28,12 +28,27 @@ class HostMatcher(object): class ProxyConfig: - def __init__(self, host='', port=8080, server_version=version.NAMEVERSION, - confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, - mode=None, upstream_server=None, http_form_in=None, http_form_out=None, - authenticator=None, ignore_hosts=[], tcp_hosts=[], - ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS): + def __init__( + self, + host='', + port=8080, + server_version=version.NAMEVERSION, + cadir=CA_DIR, + clientcerts=None, + no_upstream_cert=False, + body_size_limit=None, + mode=None, + upstream_server=None, + http_form_in=None, + http_form_out=None, + authenticator=None, + ignore_hosts=[], + tcp_hosts=[], + ciphers=None, + certs=[], + certforward=False, + ssl_ports=TRANSPARENT_SSL_PORTS + ): self.host = host self.port = port self.server_version = server_version @@ -60,8 +75,8 @@ class ProxyConfig: self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) self.authenticator = authenticator - self.confdir = os.path.expanduser(confdir) - self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) + self.cadir = os.path.expanduser(cadir) + self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) self.certforward = certforward @@ -136,7 +151,7 @@ def process_proxy_options(parser, options): return ProxyConfig( host=options.addr, port=options.port, - confdir=options.confdir, + cadir=options.cadir, clientcerts=options.clientcerts, no_upstream_cert=options.no_upstream_cert, body_size_limit=body_size_limit, -- cgit v1.2.3 From 6c1dc4522d7bf83c7b6c289f11f5a33d5b9a018f Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 16:29:38 +1300 Subject: Refactor command-line argument definition - Argument definitions live in cmdline.py - Parsing and initial processing lives in main.py --- libmproxy/cmdline.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++--- libmproxy/main.py | 125 +++++---------------------------------------------- 2 files changed, 124 insertions(+), 122 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 09e25ada..99e977d4 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,8 +1,9 @@ from __future__ import absolute_import import re -from configargparse import ArgumentTypeError +import configargparse +import argparse from netlib import http -from . import filt, utils +from . import filt, utils, version from .proxy import config APP_HOST = "mitm.it" @@ -103,7 +104,9 @@ def parse_server_spec(url): p = http.parse_url(normalized_url) if not p or not p[1]: - raise ArgumentTypeError("Invalid server specification: %s" % url) + raise argparse.ArgumentTypeError( + "Invalid server specification: %s" % url + ) if url.lower().startswith("https2http"): ssl = [True, False] @@ -132,17 +135,19 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise argparse.ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise argparse.ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise ArgumentTypeError("Could not read replace file: %s" % path) + raise argparse.ArgumentTypeError( + "Could not read replace file: %s" % path + ) reps.append((patt, rex, v)) setheaders = [] @@ -150,7 +155,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise argparse.ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -508,3 +513,105 @@ def common_options(parser): ) config.ssl_option_group(parser) + + +def mitmproxy(): + # Don't import libmproxy.console for mitmdump, urwid is not available on all + # platforms. + from .console import palettes + + parser = configargparse.ArgumentParser(usage="%(prog)s [options]") + parser.add_argument( + '--version', + action='version', + version=version.NAMEVERSION + ) + common_options(parser) + parser.add_argument( + "--palette", type=str, default="dark", + action="store", dest="palette", + help="Select color palette: " + ", ".join(palettes.palettes.keys()) + ) + parser.add_argument( + "-e", + action="store_true", dest="eventlog", + help="Show event log." + ) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + + return parser + + +def mitmdump(): + parser = configargparse.ArgumentParser(usage="%(prog)s [options] [filter]") + + parser.add_argument( + '--version', + action= 'version', + version= "mitmdump" + " " + version.VERSION + ) + common_options(parser) + parser.add_argument( + "--keepserving", + action= "store_true", dest="keepserving", default=False, + help= """ + Continue serving after client playback or file read. We exit by + default. + """ + ) + parser.add_argument( + "-d", + action="count", dest="flow_detail", default=1, + help="Increase flow detail display level. Can be passed multiple times." + ) + parser.add_argument('args', nargs=argparse.REMAINDER) + return parser + + +def mitmweb(): + parser = configargparse.ArgumentParser(usage="%(prog)s [options]") + parser.add_argument( + '--version', + action='version', + version="mitmweb" + " " + version.VERSION + ) + + group = parser.add_argument_group("Mitmweb") + group.add_argument( + "--wport", + action="store", type=int, dest="wport", default=8081, + metavar="PORT", + help="Mitmweb port." + ) + group.add_argument( + "--wiface", + action="store", dest="wiface", default="127.0.0.1", + metavar="IFACE", + help="Mitmweb interface." + ) + group.add_argument( + "--wdebug", + action="store_true", dest="wdebug", + help="Turn on mitmweb debugging" + ) + + common_options(parser) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + return parser + diff --git a/libmproxy/main.py b/libmproxy/main.py index 9cad5dcc..ffa012d3 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -1,6 +1,4 @@ from __future__ import print_function, absolute_import -import configargparse -import argparse import os import signal import sys @@ -74,39 +72,13 @@ def get_server(dummy_server, options): sys.exit(1) -def mitmproxy_cmdline(): - # Don't import libmproxy.console for mitmdump, urwid is not available on all - # platforms. +def mitmproxy(): # pragma: nocover from . import console - from .console import palettes - parser = configargparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument( - '--version', - action='version', - version=version.NAMEVERSION - ) - cmdline.common_options(parser) - parser.add_argument( - "--palette", type=str, default="dark", - action="store", dest="palette", - help="Select color palette: " + ", ".join(palettes.palettes.keys()) - ) - parser.add_argument( - "-e", - action="store_true", dest="eventlog", - help="Show event log." - ) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + check_versions() + assert_utf8_env() + parser = cmdline.mitmproxy() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -117,15 +89,6 @@ def mitmproxy_cmdline(): console_options.eventlog = options.eventlog console_options.intercept = options.intercept - return console_options, proxy_config - - -def mitmproxy(): # pragma: nocover - from . import console - - check_versions() - assert_utf8_env() - console_options, proxy_config = mitmproxy_cmdline() server = get_server(console_options.no_server, proxy_config) m = console.ConsoleMaster(server, console_options) @@ -135,32 +98,12 @@ def mitmproxy(): # pragma: nocover pass -def mitmdump_cmdline(): +def mitmdump(): # pragma: nocover from . import dump - parser = configargparse.ArgumentParser(usage="%(prog)s [options] [filter]") - - parser.add_argument( - '--version', - action= 'version', - version= "mitmdump" + " " + version.VERSION - ) - cmdline.common_options(parser) - parser.add_argument( - "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ - Continue serving after client playback or file read. We exit by - default. - """ - ) - parser.add_argument( - "-d", - action="count", dest="flow_detail", default=1, - help="Increase flow detail display level. Can be passed multiple times." - ) - parser.add_argument('args', nargs=argparse.REMAINDER) + check_versions() + parser = cmdline.mitmdump() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -172,14 +115,6 @@ def mitmdump_cmdline(): dump_options.keepserving = options.keepserving dump_options.filtstr = " ".join(options.args) if options.args else None - return dump_options, proxy_config - - -def mitmdump(): # pragma: nocover - from . import dump - - check_versions() - dump_options, proxy_config = mitmdump_cmdline() server = get_server(dump_options.no_server, proxy_config) try: @@ -197,44 +132,11 @@ def mitmdump(): # pragma: nocover pass -def mitmweb_cmdline(): +def mitmweb(): # pragma: nocover from . import web - parser = configargparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument( - '--version', - action='version', - version="mitmweb" + " " + version.VERSION - ) - - group = parser.add_argument_group("Mitmweb") - group.add_argument( - "--wport", - action="store", type=int, dest="wport", default=8081, - metavar="PORT", - help="Mitmweb port." - ) - group.add_argument( - "--wiface", - action="store", dest="wiface", default="127.0.0.1", - metavar="IFACE", - help="Mitmweb interface." - ) - group.add_argument( - "--wdebug", - action="store_true", dest="wdebug", - help="Turn on mitmweb debugging" - ) - cmdline.common_options(parser) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + check_versions() + parser = cmdline.mitmweb() options = parser.parse_args() if options.quiet: @@ -246,14 +148,7 @@ def mitmweb_cmdline(): web_options.wdebug = options.wdebug web_options.wiface = options.wiface web_options.wport = options.wport - return web_options, proxy_config - - -def mitmweb(): # pragma: nocover - from . import web - check_versions() - web_options, proxy_config = mitmweb_cmdline() server = get_server(web_options.no_server, proxy_config) m = web.WebMaster(server, web_options) -- cgit v1.2.3 From 09c503563ad2e42812bf8043aedd9ecf980babf6 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 17:25:05 +1300 Subject: Enable config file parsing We support 4 different config files: ~/.mitmproxy/common.conf: Options that are common to all tools ~/.mitmproxy/mitmproxy.conf: Options for mitmproxy ~/.mitmproxy/mitmdump.conf: Options for mitmdump ~/.mitmproxy/mitmweb.conf: Options for mitmweb Options in the tool-specific config files over-ride options in common.conf. If a non-common option is put in common.conf, an error will be raised if a non-supporting tool is used. --- libmproxy/cmdline.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 99e977d4..20dd0b6a 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import os import re import configargparse import argparse @@ -311,7 +312,7 @@ def common_options(parser): help="Don't start a proxy server." ) group.add_argument( - "-p", + "-p", "--port", action="store", type=int, dest="port", default=8080, help="Proxy service port." ) @@ -520,7 +521,16 @@ def mitmproxy(): # platforms. from .console import palettes - parser = configargparse.ArgumentParser(usage="%(prog)s [options]") + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmproxy.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) parser.add_argument( '--version', action='version', @@ -551,7 +561,16 @@ def mitmproxy(): def mitmdump(): - parser = configargparse.ArgumentParser(usage="%(prog)s [options] [filter]") + parser = configargparse.ArgumentParser( + usage="%(prog)s [options] [filter]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmdump.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) parser.add_argument( '--version', @@ -577,7 +596,16 @@ def mitmdump(): def mitmweb(): - parser = configargparse.ArgumentParser(usage="%(prog)s [options]") + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmweb.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) parser.add_argument( '--version', action='version', -- cgit v1.2.3 From 7d76f3e992409caf1997b031a910b8bfdf3fc2a3 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 17:41:04 +1300 Subject: Make sure all command-line arguments have a long form ... so they can be used in config files --- libmproxy/cmdline.py | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 20dd0b6a..27847e75 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -209,17 +209,17 @@ def common_options(parser): help="Use the Host header to construct URLs for display." ) parser.add_argument( - "-q", + "-q", "--quiet", action="store_true", dest="quiet", help="Quiet." ) parser.add_argument( - "-r", + "-r", "--read-flows", action="store", dest="rfile", default=None, help="Read flows from file." ) parser.add_argument( - "-s", + "-s", "--script", action="append", type=str, dest="scripts", default=[], metavar='"script.py --bar"', help=""" @@ -228,7 +228,7 @@ def common_options(parser): """ ) parser.add_argument( - "-t", + "-t", "--stickycookie", action="store", dest="stickycookie_filt", default=None, @@ -236,27 +236,27 @@ def common_options(parser): help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( - "-u", + "-u", "--stickyauth", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", help="Set sticky auth filter. Matched against requests." ) parser.add_argument( - "-v", + "-v", "--verbose", action="store_const", dest="verbose", default=1, const=2, help="Increase event log verbosity." ) parser.add_argument( - "-w", + "-w", "--wfile", action="store", dest="wfile", default=None, help="Write flows to file." ) parser.add_argument( - "-z", + "-z", "--anticomp", action="store_true", dest="anticomp", default=False, help="Try to convince servers to send us un-compressed data." ) parser.add_argument( - "-Z", + "-Z", "--body-size-limit", action="store", dest="body_size_limit", default=None, metavar="SIZE", help="Byte size limit of HTTP request and response bodies." @@ -279,7 +279,7 @@ def common_options(parser): # it's already in a different group. - our own error messages are more # helpful group.add_argument( - "-b", + "-b", "--bind-address", action="store", type=str, dest="addr", default='', help="Address to bind proxy to (defaults to all interfaces)" ) @@ -307,7 +307,7 @@ def common_options(parser): """ ) group.add_argument( - "-n", + "-n", "--no-server", action="store_true", dest="no_server", help="Don't start a proxy server." ) @@ -317,7 +317,7 @@ def common_options(parser): help="Proxy service port." ) group.add_argument( - "-R", + "-R", "--reverse", action="store", type=parse_server_spec, dest="reverse_proxy", @@ -333,12 +333,12 @@ def common_options(parser): help="Set SOCKS5 proxy mode." ) group.add_argument( - "-T", + "-T", "--transparent", action="store_true", dest="transparent_proxy", default=False, help="Set transparent proxy mode." ) group.add_argument( - "-U", + "-U", "--upstream", action="store", type=parse_server_spec, dest="upstream_proxy", @@ -367,7 +367,7 @@ def common_options(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( - "-a", + "-a", "--noapp", action="store_false", dest="app", default=True, help="Disable the mitmproxy onboarding app." ) @@ -392,19 +392,19 @@ def common_options(parser): group = parser.add_argument_group("Client Replay") group.add_argument( - "-c", + "-c", "--client-replay", action="store", dest="client_replay", default=None, metavar="PATH", help="Replay client requests from a saved file." ) group = parser.add_argument_group("Server Replay") group.add_argument( - "-S", + "-S", "--server-replay", action="store", dest="server_replay", default=None, metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( - "-k", + "-k", "--kill", action="store_true", dest="kill", default=False, help="Kill extra requests during replay." ) @@ -543,7 +543,7 @@ def mitmproxy(): help="Select color palette: " + ", ".join(palettes.palettes.keys()) ) parser.add_argument( - "-e", + "-e", "--eventlog", action="store_true", dest="eventlog", help="Show event log." ) @@ -556,7 +556,6 @@ def mitmproxy(): type=str, dest="intercept", default=None, help="Intercept filter expression." ) - return parser @@ -587,7 +586,7 @@ def mitmdump(): """ ) parser.add_argument( - "-d", + "-d", "--detail", action="count", dest="flow_detail", default=1, help="Increase flow detail display level. Can be passed multiple times." ) -- cgit v1.2.3 From 5af7c9ebf4c61f2397ea18c132f4253362ca075d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 17:47:39 +1300 Subject: Exclude main.py from coverage analysis --- libmproxy/main.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/main.py b/libmproxy/main.py index ffa012d3..e5b7f56b 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -8,6 +8,9 @@ from .proxy import process_proxy_options, ProxyServerError from .proxy.server import DummyServer, ProxyServer +# This file is not included in coverage analysis or tests - anything that can be +# tested should live elsewhere. + def check_versions(): """ Having installed a wrong version of pyOpenSSL or netlib is unfortunately a -- cgit v1.2.3 From 23a4f159fd1cd529743fc445f3747062fc318534 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 17:51:21 +1300 Subject: Remove last vestiges of argparse --- libmproxy/cmdline.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 27847e75..83aac790 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import os import re import configargparse -import argparse from netlib import http from . import filt, utils, version from .proxy import config @@ -105,7 +104,7 @@ def parse_server_spec(url): p = http.parse_url(normalized_url) if not p or not p[1]: - raise argparse.ArgumentTypeError( + raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) @@ -136,17 +135,17 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise argparse.ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise argparse.ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise argparse.ArgumentTypeError( + raise configargparse.ArgumentTypeError( "Could not read replace file: %s" % path ) reps.append((patt, rex, v)) @@ -156,7 +155,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise argparse.ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -590,7 +589,7 @@ def mitmdump(): action="count", dest="flow_detail", default=1, help="Increase flow detail display level. Can be passed multiple times." ) - parser.add_argument('args', nargs=argparse.REMAINDER) + parser.add_argument('args', nargs="...") return parser -- cgit v1.2.3 From aa77a52a069e832236495f2aa7bfdbc90f26b59c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 15 Nov 2014 17:58:38 +1300 Subject: One common --version flag --- libmproxy/cmdline.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 83aac790..b892f1fd 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -188,6 +188,11 @@ def get_common_options(options): def common_options(parser): + parser.add_argument( + '--version', + action= 'version', + version= "%(prog)s" + " " + version.VERSION + ) parser.add_argument( "--anticache", action="store_true", dest="anticache", default=False, @@ -530,11 +535,6 @@ def mitmproxy(): add_config_file_help = True, add_env_var_help = True ) - parser.add_argument( - '--version', - action='version', - version=version.NAMEVERSION - ) common_options(parser) parser.add_argument( "--palette", type=str, default="dark", @@ -570,11 +570,6 @@ def mitmdump(): add_env_var_help = True ) - parser.add_argument( - '--version', - action= 'version', - version= "mitmdump" + " " + version.VERSION - ) common_options(parser) parser.add_argument( "--keepserving", @@ -604,11 +599,6 @@ def mitmweb(): add_config_file_help = True, add_env_var_help = True ) - parser.add_argument( - '--version', - action='version', - version="mitmweb" + " " + version.VERSION - ) group = parser.add_argument_group("Mitmweb") group.add_argument( -- cgit v1.2.3 From ec235941919c184641f9ab30f2df13ab7fea0414 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 19 Nov 2014 01:27:20 +0100 Subject: add sni support to LiveConnection.change_server --- libmproxy/protocol/primitives.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 1bf7f832..3be1cc45 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -174,7 +174,7 @@ class LiveConnection(object): self._backup_server_conn = None """@type: libmproxy.proxy.connection.ServerConnection""" - def change_server(self, address, ssl=None, force=False, persistent_change=False): + def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False): """ Change the server connection to the specified address. @returns: @@ -183,7 +183,14 @@ class LiveConnection(object): """ address = netlib.tcp.Address.wrap(address) - ssl_mismatch = (ssl is not None and ssl != self.c.server_conn.ssl_established) + ssl_mismatch = ( + ssl is not None and + ( + ssl != self.c.server_conn.ssl_established + or + (sni is not None and sni != self.c.sni) + ) + ) address_mismatch = (address != self.c.server_conn.address) if persistent_change: @@ -212,6 +219,8 @@ class LiveConnection(object): self.c.set_server_address(address) self.c.establish_server_connection(ask=False) + if sni: + self.c.sni = sni if ssl: self.c.establish_ssl(server=True) return True -- cgit v1.2.3 From f7c5385679a6e10f707b744242dc8edcf1028bf7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 22 Nov 2014 15:27:43 +0100 Subject: retain raw filter str on filt objects --- libmproxy/filt.py | 4 +++- libmproxy/flow.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 7d2bd737..5d259096 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -343,7 +343,9 @@ bnf = _make() def parse(s): try: - return bnf.parseString(s, parseAll=True)[0] + filt = bnf.parseString(s, parseAll=True)[0] + filt.pattern = s + return filt except pp.ParseException: return None except ValueError: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 00713698..5abcb1ab 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -346,11 +346,13 @@ class State(object): # These are compiled filt expressions: self._limit = None self.intercept = None - self._limit_txt = None @property def limit_txt(self): - return self._limit_txt + if self.filt: + return self.filt.pattern + else: + return None def flow_count(self): return len(self._flow_list) @@ -407,10 +409,8 @@ class State(object): if not f: return "Invalid filter expression." self._limit = f - self._limit_txt = txt else: self._limit = None - self._limit_txt = None self.recalculate_view() def set_intercept(self, txt): -- cgit v1.2.3 From 47a78e3c729f4ddb7971b72bfae30140562f4dd6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 23 Nov 2014 15:46:17 +0100 Subject: fix limit_txt, fix #412 --- libmproxy/flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 5abcb1ab..a6bf17d8 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -349,8 +349,8 @@ class State(object): @property def limit_txt(self): - if self.filt: - return self.filt.pattern + if self._limit: + return self._limit.pattern else: return None -- cgit v1.2.3 From 3887e7ed29db88fd9f18d42013346c5dce5aa083 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Nov 2014 04:56:17 +0100 Subject: fix error html --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 26a94040..386a3666 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1198,7 +1198,7 @@ class HTTPHandler(ProtocolHandler): %d %s - + %s """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) -- cgit v1.2.3 From 56f1278d1a0233620289d2fa4173449ed108a73e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 28 Nov 2014 17:52:54 +0100 Subject: fix #413 --- libmproxy/console/grideditor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index 72c1e4a0..438d0ad7 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -123,7 +123,6 @@ class GridWalker(urwid.ListWalker): except ValueError: self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000) return - errors = self.lst[self.focus][1] emsg = self.editor.is_error(self.focus_col, val) if emsg: @@ -322,9 +321,11 @@ class GridEditor(common.WWrap): elif key == "d": self.walker.delete_focus() elif key == "r": - self.master.path_prompt("Read file: ", "", self.read_file) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read file: ", "", self.read_file) elif key == "R": - self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) elif key == "e": o = self.walker.get_current_value() if o is not None: -- cgit v1.2.3 From 992536c2bc0afa5da81e82cfcd8953663559ff59 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 1 Dec 2014 02:28:03 +0100 Subject: make header processing configurable by inline scripts, refs #340 --- libmproxy/protocol/http.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 386a3666..89af85b0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -419,17 +419,19 @@ class HTTPRequest(HTTPMessage): raise http.HttpError(400, "Invalid request form") return request_line + # This list is adopted legacy code. + # We probably don't need to strip off keep-alive. + _headers_to_strip_off = ['Proxy-Connection', + 'Keep-Alive', + 'Connection', + 'Transfer-Encoding', + 'Upgrade'] + def _assemble_headers(self): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Keep-Alive', - 'Connection', - 'Transfer-Encoding']: + for k in self._headers_to_strip_off: del headers[k] - if headers["Upgrade"] == ["h2c"]: - # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http - del headers["Upgrade"] - if not 'host' in headers and self.scheme and self.host and self.port: + if 'host' not in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, self.host, self.port)] @@ -750,11 +752,13 @@ class HTTPResponse(HTTPMessage): return 'HTTP/%s.%s %s %s' % \ (self.httpversion[0], self.httpversion[1], self.code, self.msg) + _headers_to_strip_off = ['Proxy-Connection', + 'Alternate-Protocol', + 'Alt-Svc'] + def _assemble_headers(self, preserve_transfer_encoding=False): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Alternate-Protocol', - 'Alt-Svc']: + for k in self._headers_to_strip_off: del headers[k] if not preserve_transfer_encoding: del headers['Transfer-Encoding'] -- cgit v1.2.3 From 5b1fefee9bf8564b32a1137975cb181d54ef6dff Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 1 Dec 2014 03:04:48 +0100 Subject: add inline script example for websocket passthrough, fix #340 --- libmproxy/protocol/http.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 89af85b0..87af8e6d 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1401,6 +1401,12 @@ class HTTPHandler(ProtocolHandler): # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards. # If we don't delegate to TCP mode, we should always negotiate a SSL connection. + # + # FIXME: + # Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80 + # if an explicit proxy is configured and a websocket connection should be established. + # We don't support websocket at the moment, so it fails anyway, but we should come up with + # a better solution to this if we start to support WebSockets. should_establish_ssl = ( address.port in self.c.config.ssl_ports or -- cgit v1.2.3 From 591ed0b41f91e2c688046de68d42115004328a96 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 4 Dec 2014 00:29:15 +0100 Subject: fix HTTPResponse creation --- libmproxy/console/flowview.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3dceff70..1ec57a4e 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -574,9 +574,8 @@ class FlowView(common.WWrap): else: if not self.flow.response: self.flow.response = HTTPResponse( - self.flow.request, self.flow.request.httpversion, - 200, "OK", flow.ODictCaseless(), "", None + 200, "OK", flow.ODictCaseless(), "" ) self.flow.response.reply = controller.DummyReply() conn = self.flow.response -- cgit v1.2.3 From 31925dc9bedde9fcb388d5254b43ac90a12d4eaf Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 8 Dec 2014 17:01:47 +0100 Subject: fix #419 --- libmproxy/onboarding/templates/index.html | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html index 50cfd5db..65fda5d2 100644 --- a/libmproxy/onboarding/templates/index.html +++ b/libmproxy/onboarding/templates/index.html @@ -1,5 +1,5 @@ {% extends "frame.html" %} -{% block body %} +{% block body %}

Click to install the mitmproxy certificate:

@@ -23,4 +23,13 @@ +
+
+ Other mitmproxy users cannot intercept your connection. +
+
+ This page is served by your local mitmproxy instance. The certificate you are about to install has been uniquely generated on mitmproxy's first run and is not shared + between mitmproxy installations. +
+ {% endblock %} -- cgit v1.2.3 From b95f0c997127bb0516cde0609c83c7d2af5ccfbc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 8 Dec 2014 17:17:37 +0100 Subject: fix #411 --- libmproxy/protocol/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 87af8e6d..49f5e8c0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1043,7 +1043,7 @@ class HTTPHandler(ProtocolHandler): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.c.channel.ask("responseheaders", flow) - if flow == KILL: + if flow is None or flow == KILL: raise KillSignal() else: # now get the rest of the request body, if body still needs to be @@ -1086,11 +1086,11 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.process_server_address(flow) # The inline script may have changed request.host - if request_reply is None or request_reply == KILL: raise KillSignal() + self.process_server_address(flow) # The inline script may have changed request.host + if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: -- cgit v1.2.3