diff options
| -rw-r--r-- | README.mkd | 8 | ||||
| -rw-r--r-- | doc-src/features/reverseproxy.html | 26 | ||||
| -rwxr-xr-x | examples/flowbasic | 2 | ||||
| -rw-r--r-- | examples/ignore_websocket.py | 34 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 14 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 3 | ||||
| -rw-r--r-- | libmproxy/console/grideditor.py | 7 | ||||
| -rw-r--r-- | libmproxy/dump.py | 8 | ||||
| -rw-r--r-- | libmproxy/onboarding/templates/index.html | 11 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 40 | ||||
| -rw-r--r-- | test/test_dump.py | 11 | ||||
| -rw-r--r-- | test/test_examples.py | 3 |
12 files changed, 126 insertions, 41 deletions
@@ -72,8 +72,7 @@ This installs the latest GitHub versions of mitmproxy, netlib and pathod into `m The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with: -` -pip install "mitmproxy[dev]""` +`pip install "mitmproxy[dev]"` Please ensure that all patches are accompanied by matching changes in the test @@ -85,8 +84,3 @@ suite. The project maintains 100% test coverage. Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this: `cshape doc-src doc` - - - - - diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index 1c57f0b2..5ef4efc5 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -25,4 +25,28 @@ service uses HTTP like this: https2http://hostname:port - +### Host Header + +In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this +may lead to issues with public web servers. For example, consider the following scenario: + + $ python mitmdump -d -R http://example.com/ & + $ curl http://localhost:8080/ + + >> GET https://example.com/ + Host: localhost:8080 + User-Agent: curl/7.35.0 + [...] + + << 404 Not Found 345B + +Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br> +There are two ways to solve this: +<ol> + <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li> + <li> + Instruct mitmproxy to rewrite the host header by passing <kbd>‑‑setheader :~q:Host:example.com</kbd>. + However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application + to bypass the proxy. + </li> +</ol>
\ No newline at end of file diff --git a/examples/flowbasic b/examples/flowbasic index 41402b0c..c71debc9 100755 --- a/examples/flowbasic +++ b/examples/flowbasic @@ -36,7 +36,7 @@ class MyMaster(flow.FlowMaster): config = proxy.ProxyConfig( port=8080, - confdir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file. + cadir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file. ) state = flow.State() server = ProxyServer(config) diff --git a/examples/ignore_websocket.py b/examples/ignore_websocket.py new file mode 100644 index 00000000..48093951 --- /dev/null +++ b/examples/ignore_websocket.py @@ -0,0 +1,34 @@ +# This script makes mitmproxy switch to passthrough mode for all HTTP +# responses with "Connection: Upgrade" header. This is useful to make +# WebSockets work in untrusted environments. +# +# Note: Chrome (and possibly other browsers), when explicitly configured +# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request +# to the proxy before they initiate the websocket connection. +# To make WebSockets work in these cases, supply +# `--ignore :80$` as an additional parameter. +# (see http://mitmproxy.org/doc/features/passthrough.html) + +from libmproxy.protocol.http import HTTPRequest +from libmproxy.protocol.tcp import TCPHandler +from libmproxy.protocol import KILL +from libmproxy.script import concurrent + + +def start(context, argv): + HTTPRequest._headers_to_strip_off.remove("Connection") + HTTPRequest._headers_to_strip_off.remove("Upgrade") + + +def done(context): + HTTPRequest._headers_to_strip_off.append("Connection") + HTTPRequest._headers_to_strip_off.append("Upgrade") + +@concurrent +def response(context, flow): + if flow.response.headers.get_first("Connection", None) == "Upgrade": + # We need to send the response manually now... + flow.client_conn.send(flow.response.assemble()) + # ...and then delegate to tcp passthrough. + TCPHandler(flow.live.c, log=False).handle_messages() + flow.reply(KILL)
\ No newline at end of file diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index b892f1fd..bf5add33 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -179,7 +179,7 @@ def get_common_options(options): stickyauth=stickyauth, stream_large_bodies=stream_large_bodies, showhost=options.showhost, - wfile=options.wfile, + outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, replay_ignore_content = options.replay_ignore_content, @@ -249,11 +249,17 @@ def common_options(parser): action="store_const", dest="verbose", default=1, const=2, help="Increase event log verbosity." ) - parser.add_argument( + outfile = parser.add_mutually_exclusive_group() + outfile.add_argument( "-w", "--wfile", - action="store", dest="wfile", default=None, + action="store", dest="outfile", type=lambda f: (f, "wb"), help="Write flows to file." ) + outfile.add_argument( + "-a", "--afile", + action="store", dest="outfile", type=lambda f: (f, "ab"), + help="Append flows to file." + ) parser.add_argument( "-z", "--anticomp", action="store_true", dest="anticomp", default=False, @@ -371,7 +377,7 @@ def common_options(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( - "-a", "--noapp", + "--noapp", action="store_false", dest="app", default=True, help="Disable the mitmproxy onboarding app." ) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3dceff70..1ec57a4e 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -574,9 +574,8 @@ class FlowView(common.WWrap): else: if not self.flow.response: self.flow.response = HTTPResponse( - self.flow.request, self.flow.request.httpversion, - 200, "OK", flow.ODictCaseless(), "", None + 200, "OK", flow.ODictCaseless(), "" ) self.flow.response.reply = controller.DummyReply() conn = self.flow.response diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index 72c1e4a0..438d0ad7 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -123,7 +123,6 @@ class GridWalker(urwid.ListWalker): except ValueError: self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000) return - errors = self.lst[self.focus][1] emsg = self.editor.is_error(self.focus_col, val) if emsg: @@ -322,9 +321,11 @@ class GridEditor(common.WWrap): elif key == "d": self.walker.delete_focus() elif key == "r": - self.master.path_prompt("Read file: ", "", self.read_file) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read file: ", "", self.read_file) elif key == "R": - self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) elif key == "e": o = self.walker.get_current_value() if o is not None: diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 0d9432c9..8f260745 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -36,7 +36,7 @@ class Options(object): "stickyauth", "stream_large_bodies", "verbosity", - "wfile", + "outfile", "replay_ignore_content", "replay_ignore_params", ] @@ -92,10 +92,10 @@ class DumpMaster(flow.FlowMaster): if options.stickyauth: self.set_stickyauth(options.stickyauth) - if options.wfile: - path = os.path.expanduser(options.wfile) + if options.outfile: + path = os.path.expanduser(options.outfile[0]) try: - f = file(path, "wb") + f = file(path, options.outfile[1]) self.start_stream(f, self.filt) except IOError, v: raise DumpError(v.strerror) diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html index 50cfd5db..65fda5d2 100644 --- a/libmproxy/onboarding/templates/index.html +++ b/libmproxy/onboarding/templates/index.html @@ -1,5 +1,5 @@ {% extends "frame.html" %} -{% block body %} +{% block body %} <center> <h2> Click to install the mitmproxy certificate: </h2> @@ -23,4 +23,13 @@ </div> </div> +<hr/> +<div class="text-center"> + Other mitmproxy users cannot intercept your connection. +</div> +<div class="text-center text-muted"> + This page is served by your local mitmproxy instance. The certificate you are about to install has been uniquely generated on mitmproxy's first run and is not shared + between mitmproxy installations. +</div> + {% endblock %} diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index b32a55ed..d3945579 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -422,17 +422,19 @@ class HTTPRequest(HTTPMessage): raise http.HttpError(400, "Invalid request form") return request_line + # This list is adopted legacy code. + # We probably don't need to strip off keep-alive. + _headers_to_strip_off = ['Proxy-Connection', + 'Keep-Alive', + 'Connection', + 'Transfer-Encoding', + 'Upgrade'] + def _assemble_headers(self): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Keep-Alive', - 'Connection', - 'Transfer-Encoding']: + for k in self._headers_to_strip_off: del headers[k] - if headers["Upgrade"] == ["h2c"]: - # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http - del headers["Upgrade"] - if not 'host' in headers and self.scheme and self.host and self.port: + if 'host' not in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, self.host, self.port)] @@ -753,11 +755,13 @@ class HTTPResponse(HTTPMessage): return 'HTTP/%s.%s %s %s' % \ (self.httpversion[0], self.httpversion[1], self.code, self.msg) + _headers_to_strip_off = ['Proxy-Connection', + 'Alternate-Protocol', + 'Alt-Svc'] + def _assemble_headers(self, preserve_transfer_encoding=False): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Alternate-Protocol', - 'Alt-Svc']: + for k in self._headers_to_strip_off: del headers[k] if not preserve_transfer_encoding: del headers['Transfer-Encoding'] @@ -1042,7 +1046,7 @@ class HTTPHandler(ProtocolHandler): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.c.channel.ask("responseheaders", flow) - if flow == KILL: + if flow is None or flow == KILL: raise KillSignal() else: # now get the rest of the request body, if body still needs to be @@ -1085,11 +1089,11 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.process_server_address(flow) # The inline script may have changed request.host - if request_reply is None or request_reply == KILL: raise KillSignal() + self.process_server_address(flow) # The inline script may have changed request.host + if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: @@ -1201,7 +1205,7 @@ class HTTPHandler(ProtocolHandler): <head> <title>%d %s</title> </head> - <body %s</body> + <body>%s</body> </html> """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) @@ -1400,6 +1404,12 @@ class HTTPHandler(ProtocolHandler): # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards. # If we don't delegate to TCP mode, we should always negotiate a SSL connection. + # + # FIXME: + # Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80 + # if an explicit proxy is configured and a websocket connection should be established. + # We don't support websocket at the moment, so it fails anyway, but we should come up with + # a better solution to this if we start to support WebSockets. should_establish_ssl = ( address.port in self.c.config.ssl_ports or diff --git a/test/test_dump.py b/test/test_dump.py index e9cb4d33..aa91d262 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -143,9 +143,16 @@ class TestDumpMaster: def test_write(self): with tutils.tmpdir() as d: p = os.path.join(d, "a") - self._dummy_cycle(1, None, "", wfile=p, verbosity=0) + self._dummy_cycle(1, None, "", outfile=(p,"wb"), verbosity=0) assert len(list(flow.FlowReader(open(p,"rb")).stream())) == 1 + def test_write_append(self): + with tutils.tmpdir() as d: + p = os.path.join(d, "a.append") + self._dummy_cycle(1, None, "", outfile=(p,"wb"), verbosity=0) + self._dummy_cycle(1, None, "", outfile=(p,"ab"), verbosity=0) + assert len(list(flow.FlowReader(open(p,"rb")).stream())) == 2 + def test_write_err(self): tutils.raises( dump.DumpError, @@ -153,7 +160,7 @@ class TestDumpMaster: 1, None, "", - wfile = "nonexistentdir/foo" + outfile = ("nonexistentdir/foo", "wb") ) def test_script(self): diff --git a/test/test_examples.py b/test/test_examples.py index a5a212cd..deb97b49 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -16,4 +16,5 @@ def test_load_scripts(): f += " foo" # one argument required if "modify_response_body" in f: f += " foo bar" # two arguments required - script.Script(f, tmaster) # Loads the script file.
\ No newline at end of file + s = script.Script(f, tmaster) # Loads the script file. + s.unload()
\ No newline at end of file |