diff options
| -rw-r--r-- | doc-src/_nav.html | 1 | ||||
| -rw-r--r-- | doc-src/dev/index.py | 1 | ||||
| -rw-r--r-- | doc-src/dev/sslkeylogfile.html | 8 | ||||
| -rw-r--r-- | doc-src/features/responsestreaming.html | 6 | ||||
| -rw-r--r-- | examples/README | 1 | ||||
| -rw-r--r-- | examples/dns_spoofing.py | 35 | ||||
| -rw-r--r-- | examples/stream_modify.py | 22 | ||||
| -rw-r--r-- | libmproxy/flow.py | 2 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 17 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 52 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 31 | ||||
| -rw-r--r-- | setup.py | 8 | ||||
| -rw-r--r-- | test/scripts/stream_modify.py | 7 | ||||
| -rw-r--r-- | test/test_server.py | 37 | ||||
| -rw-r--r-- | test/tservers.py | 4 |
15 files changed, 189 insertions, 43 deletions
diff --git a/doc-src/_nav.html b/doc-src/_nav.html index 6c3afbe1..69175c0c 100644 --- a/doc-src/_nav.html +++ b/doc-src/_nav.html @@ -56,4 +56,5 @@ <li class="nav-header">Hacking</li> $!nav("dev/architecture.html", this, state)!$ $!nav("dev/testing.html", this, state)!$ + $!nav("dev/sslkeylogfile.html", this, state)!$ </ul> diff --git a/doc-src/dev/index.py b/doc-src/dev/index.py index bb7872c7..0f2a6494 100644 --- a/doc-src/dev/index.py +++ b/doc-src/dev/index.py @@ -3,5 +3,6 @@ from countershape import Page pages = [ Page("testing.html", "Testing"), Page("architecture.html", "Architecture"), + Page("sslkeylogfile.html", "TLS Master Secrets"), # Page("addingviews.html", "Writing Content Views"), ] diff --git a/doc-src/dev/sslkeylogfile.html b/doc-src/dev/sslkeylogfile.html new file mode 100644 index 00000000..1826fc2e --- /dev/null +++ b/doc-src/dev/sslkeylogfile.html @@ -0,0 +1,8 @@ +The SSL master keys can be logged by mitmproxy so that external programs can decrypt TLS connections both from and to the proxy. +Key logging is enabled by setting the environment variable <samp>SSLKEYLOGFILE</samp> so that it points to a writable +text file. Recent versions of WireShark can use these log files to decrypt packets. +You can specify the key file path in WireShark via<br> +<samp>Edit → Preferences → Protocols → SSL → (Pre)-Master-Secret log filename</samp>. + + Note that <samp>SSLKEYLOGFILE</samp> is respected by other programs as well, e.g. Firefox and Chrome. +If this creates any issues, you can set <samp>MITMPROXY_SSLKEYLOGFILE</samp> alternatively.
\ No newline at end of file diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 47fafef7..6511e913 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,7 +40,6 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ - <h2>Implementation Details</h2> When response streaming is enabled, portions of the code which would have otherwise performed changes @@ -49,6 +48,11 @@ on the response body will see an empty response body instead (<code>libmproxy.pr Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding: chunked</code> header, the response will be streamed one chunk at a time. +<h2>Modifying streamed data</h2> +If the <code>.stream</code> attribute is callable, .stream will work as a hook in chunk data processing. + +$!example("examples/stream_modify.py")!$ + ### See Also - [Ignore Domains](@!urlTo("passthrough.html")!@) diff --git a/examples/README b/examples/README index 85ab272a..f24c4de7 100644 --- a/examples/README +++ b/examples/README @@ -1,6 +1,7 @@ # inline script examples add_header.py Simple script that just adds a header to every request. change_upstream_proxy.py Dynamically change the upstream proxy +dns_spoofing.py Use mitmproxy in a DNS spoofing scenario. dup_and_replay.py Duplicates each request, changes it, and then replays the modified request. iframe_injector.py Inject configurable iframe into pages. modify_form.py Modify all form submissions to add a parameter. diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py new file mode 100644 index 00000000..cfba7c54 --- /dev/null +++ b/examples/dns_spoofing.py @@ -0,0 +1,35 @@ +""" +This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect +connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the +Host header of the HTTP request. +Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't +know the actual target and cannot construct a certificate that looks valid. +Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well. +Using transparent mode is the better option most of the time. + +Usage: + mitmproxy + -p 80 + -R http://example.com/ // Used as the target location if no Host header is present + mitmproxy + -p 443 + -R https://example.com/ // Used as the target locaction if neither SNI nor host header are present. + +mitmproxy will always connect to the default location first, so it must be reachable. +As a workaround, you can spawn an arbitrary HTTP server and use that for both endpoints, e.g. +mitmproxy -p 80 -R http://localhost:8000 +mitmproxy -p 443 -R https2http://localhost:8000 +""" + + +def request(context, flow): + if flow.client_conn.ssl_established: + # TLS SNI or Host header + flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True) + + # If you use a https2http location as default destination, these attributes need to be corrected as well: + flow.request.port = 443 + flow.request.scheme = "https" + else: + # Host header + flow.request.host = flow.request.pretty_host(hostheader=True)
\ No newline at end of file diff --git a/examples/stream_modify.py b/examples/stream_modify.py new file mode 100644 index 00000000..56d26e6d --- /dev/null +++ b/examples/stream_modify.py @@ -0,0 +1,22 @@ +""" +This inline script modifies a streamed response. +If you do not need streaming, see the modify_response_body example. +Be aware that content replacement isn't trivial: + - If the transfer encoding isn't chunked, you cannot simply change the content length. + - If you want to replace all occurences of "foobar", make sure to catch the cases + where one chunk ends with [...]foo" and the next starts with "bar[...]. +""" + + +def modify(chunks): + """ + chunks is a generator that can be used to iterate over all chunks. + Each chunk is a (prefix, content, suffix) tuple. + For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n") + """ + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo", "bar"), suffix + + +def responseheaders(context, flow): + flow.response.stream = modify
\ No newline at end of file diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 14497964..43580109 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -165,7 +165,7 @@ class StreamLargeBodies(object): r.headers, is_request, flow.request.method, code ) if not (0 <= expected_size <= self.max_size): - r.stream = True + r.stream = r.stream or True # r.stream may already be a callable, which we want to preserve. class ClientPlaybackState: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 046d0b42..49310ec3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,10 +1332,19 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096): + + chunks = http.read_http_body_chunked( + self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + for chunk in chunks: for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 84893323..dfde2958 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -45,7 +45,8 @@ class ProxyConfig: authenticator=None, ignore_hosts=[], tcp_hosts=[], - ciphers=None, + ciphers_client=None, + ciphers_server=None, certs=[], certforward=False, ssl_version_client="secure", @@ -55,7 +56,8 @@ class ProxyConfig: self.host = host self.port = port self.server_version = server_version - self.ciphers = ciphers + self.ciphers_client = ciphers_client + self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit @@ -83,8 +85,8 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) self.certforward = certforward - self.openssl_client_method, self.openssl_client_options = version_to_openssl(ssl_version_client) - self.openssl_server_method, self.openssl_server_options = version_to_openssl(ssl_version_server) + self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client) + self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server) self.ssl_ports = ssl_ports @@ -188,7 +190,8 @@ def process_proxy_options(parser, options): ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, - ciphers=options.ciphers, + ciphers_client=options.ciphers_client, + ciphers_server=options.ciphers_server, certs=certs, certforward=options.certforward, ssl_version_client=options.ssl_version_client, @@ -210,19 +213,35 @@ def ssl_option_group(parser): 'Can be passed multiple times.' ) group.add_argument( + "--cert-forward", action="store_true", + dest="certforward", default=False, + help="Simply forward SSL certificates from upstream." + ) + group.add_argument( + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=None, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" + ) + group.add_argument( + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" + ) + group.add_argument( "--client-certs", action="store", type=str, dest="clientcerts", default=None, help="Client certificate directory." ) group.add_argument( - "--ciphers", action="store", - type=str, dest="ciphers", default=None, - help="SSL cipher specification." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--cert-forward", action="store_true", - dest="certforward", default=False, - help="Simply forward SSL certificates from upstream." + "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), + metavar="PORT", + help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " + "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) ) group.add_argument( "--ssl-version-client", dest="ssl_version_client", @@ -238,14 +257,3 @@ def ssl_option_group(parser): help="Set supported SSL/TLS version for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) - )
\ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index ea78d964..896dd024 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -187,8 +187,9 @@ class ConnectionHandler: self.server_conn.establish_ssl( self.config.clientcerts, sni, - method=self.config.openssl_server_method, - options=self.config.openssl_server_options + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + cipher_list=self.config.ciphers_server, ) except tcp.NetLibError as v: e = ProxyError(502, repr(v)) @@ -207,10 +208,10 @@ class ConnectionHandler: try: self.client_conn.convert_to_ssl( cert, key, - method=self.config.openssl_client_method, - options=self.config.openssl_client_options, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, handle_sni=self.handle_sni, - cipher_list=self.config.ciphers, + cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) @@ -260,11 +261,12 @@ class ConnectionHandler: sans = [] if self.server_conn.ssl_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) if upstream_cert.cn: + sans.append(host) host = upstream_cert.cn.decode("utf8").encode("idna") - sans = upstream_cert.altnames - elif self.server_conn.sni: - sans = [self.server_conn.sni] + if self.server_conn.sni: + sans.append(self.server_conn.sni) ret = self.config.certstore.get_cert(host, sans) if not ret: @@ -285,14 +287,19 @@ class ConnectionHandler: if sni != self.server_conn.sni: self.log("SNI received: %s" % sni, "debug") - self.server_reconnect(sni) # reconnect to upstream server with SNI + # We should only re-establish upstream SSL if one of the following conditions is true: + # - We established SSL with the server previously + # - We initially wanted to establish SSL with the server, + # but the server refused to negotiate without SNI. + if self.server_conn.ssl_established or hasattr(self.server_conn, "may_require_sni"): + self.server_reconnect(sni) # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( cert, key, - method=self.config.openssl_client_method, - options=self.config.openssl_client_options, - cipher_list=self.config.ciphers, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) @@ -36,6 +36,10 @@ for script in scripts: if os.name == "nt": deps.add("pydivert>=0.0.7") # Transparent proxying on Windows +console_scripts = [ + "%s = libmproxy.main:%s" % (s, s) for s in scripts +] + setup( name="mitmproxy", version=version.VERSION, @@ -65,7 +69,9 @@ setup( ], packages=find_packages(), include_package_data=True, - scripts=scripts, + entry_points={ + 'console_scripts': console_scripts + }, install_requires=list(deps), extras_require={ 'dev': [ diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py new file mode 100644 index 00000000..9a98a7ee --- /dev/null +++ b/test/scripts/stream_modify.py @@ -0,0 +1,7 @@ +def modify(chunks): + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo", "bar"), suffix + + +def responseheaders(context, flow): + flow.response.stream = modify
\ No newline at end of file diff --git a/test/test_server.py b/test/test_server.py index a611d30f..26770f29 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,5 +1,6 @@ import socket, time from libmproxy.proxy.config import HostMatcher +import libpathod from netlib import tcp, http_auth, http from libpathod import pathoc, pathod from netlib.certutils import SSLCert @@ -265,6 +266,12 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): assert self.master.state.view[-1].response.content == CONTENT_MISSING self.master.set_stream_large_bodies(None) + def test_stream_modify(self): + self.master.load_script(tutils.test_data.path("scripts/stream_modify.py")) + d = self.pathod('200:b"foo"') + assert d.content == "bar" + self.master.unload_scripts() + class TestHTTPAuth(tservers.HTTPProxTest): authenticator = http_auth.BasicProxyAuth(http_auth.PassManSingleUser("test", "test"), "realm") def test_auth(self): @@ -332,6 +339,36 @@ class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin): reverse = True +class TestHttps2Http(tservers.ReverseProxTest): + @classmethod + def get_proxy_config(cls): + d = super(TestHttps2Http, cls).get_proxy_config() + d["upstream_server"][0] = True + return d + + def pathoc(self, ssl, sni=None): + """ + Returns a connected Pathoc instance. + """ + p = libpathod.pathoc.Pathoc(("localhost", self.proxy.port), ssl=ssl, sni=sni) + p.connect() + return p + + def test_all(self): + p = self.pathoc(ssl=True) + assert p.request("get:'/p/200'").status_code == 200 + + def test_sni(self): + p = self.pathoc(ssl=True, sni="example.com") + assert p.request("get:'/p/200'").status_code == 200 + assert all("Error in handle_sni" not in msg for msg in self.proxy.log) + + def test_http(self): + p = self.pathoc(ssl=False) + assert p.request("get:'/p/200'").status_code == 400 + + + class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False diff --git a/test/tservers.py b/test/tservers.py index 37929d1a..30c8b52e 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -218,12 +218,12 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = ( + d["upstream_server"] = [ True if cls.ssl else False, True if cls.ssl else False, "127.0.0.1", cls.server.port - ) + ] d["mode"] = "reverse" return d |