diff options
| -rw-r--r-- | doc-src/_nav.html | 1 | ||||
| -rw-r--r-- | doc-src/dev/index.py | 1 | ||||
| -rw-r--r-- | doc-src/dev/sslkeylogfile.html | 8 | ||||
| -rw-r--r-- | doc-src/features/responsestreaming.html | 6 | ||||
| -rw-r--r-- | examples/stream_modify.py | 22 | ||||
| -rw-r--r-- | libmproxy/flow.py | 2 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 17 | ||||
| -rw-r--r-- | test/scripts/stream_modify.py | 7 | ||||
| -rw-r--r-- | test/test_server.py | 6 |
9 files changed, 64 insertions, 6 deletions
diff --git a/doc-src/_nav.html b/doc-src/_nav.html index 6c3afbe1..69175c0c 100644 --- a/doc-src/_nav.html +++ b/doc-src/_nav.html @@ -56,4 +56,5 @@ <li class="nav-header">Hacking</li> $!nav("dev/architecture.html", this, state)!$ $!nav("dev/testing.html", this, state)!$ + $!nav("dev/sslkeylogfile.html", this, state)!$ </ul> diff --git a/doc-src/dev/index.py b/doc-src/dev/index.py index bb7872c7..0f2a6494 100644 --- a/doc-src/dev/index.py +++ b/doc-src/dev/index.py @@ -3,5 +3,6 @@ from countershape import Page pages = [ Page("testing.html", "Testing"), Page("architecture.html", "Architecture"), + Page("sslkeylogfile.html", "TLS Master Secrets"), # Page("addingviews.html", "Writing Content Views"), ] diff --git a/doc-src/dev/sslkeylogfile.html b/doc-src/dev/sslkeylogfile.html new file mode 100644 index 00000000..1826fc2e --- /dev/null +++ b/doc-src/dev/sslkeylogfile.html @@ -0,0 +1,8 @@ +The SSL master keys can be logged by mitmproxy so that external programs can decrypt TLS connections both from and to the proxy. +Key logging is enabled by setting the environment variable <samp>SSLKEYLOGFILE</samp> so that it points to a writable +text file. Recent versions of WireShark can use these log files to decrypt packets. +You can specify the key file path in WireShark via<br> +<samp>Edit → Preferences → Protocols → SSL → (Pre)-Master-Secret log filename</samp>. + + Note that <samp>SSLKEYLOGFILE</samp> is respected by other programs as well, e.g. Firefox and Chrome. +If this creates any issues, you can set <samp>MITMPROXY_SSLKEYLOGFILE</samp> alternatively.
\ No newline at end of file diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 47fafef7..6511e913 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,7 +40,6 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ - <h2>Implementation Details</h2> When response streaming is enabled, portions of the code which would have otherwise performed changes @@ -49,6 +48,11 @@ on the response body will see an empty response body instead (<code>libmproxy.pr Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding: chunked</code> header, the response will be streamed one chunk at a time. +<h2>Modifying streamed data</h2> +If the <code>.stream</code> attribute is callable, .stream will work as a hook in chunk data processing. + +$!example("examples/stream_modify.py")!$ + ### See Also - [Ignore Domains](@!urlTo("passthrough.html")!@) diff --git a/examples/stream_modify.py b/examples/stream_modify.py new file mode 100644 index 00000000..56d26e6d --- /dev/null +++ b/examples/stream_modify.py @@ -0,0 +1,22 @@ +""" +This inline script modifies a streamed response. +If you do not need streaming, see the modify_response_body example. +Be aware that content replacement isn't trivial: + - If the transfer encoding isn't chunked, you cannot simply change the content length. + - If you want to replace all occurences of "foobar", make sure to catch the cases + where one chunk ends with [...]foo" and the next starts with "bar[...]. +""" + + +def modify(chunks): + """ + chunks is a generator that can be used to iterate over all chunks. + Each chunk is a (prefix, content, suffix) tuple. + For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n") + """ + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo", "bar"), suffix + + +def responseheaders(context, flow): + flow.response.stream = modify
\ No newline at end of file diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 14497964..43580109 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -165,7 +165,7 @@ class StreamLargeBodies(object): r.headers, is_request, flow.request.method, code ) if not (0 <= expected_size <= self.max_size): - r.stream = True + r.stream = r.stream or True # r.stream may already be a callable, which we want to preserve. class ClientPlaybackState: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 046d0b42..49310ec3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,10 +1332,19 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096): + + chunks = http.read_http_body_chunked( + self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + for chunk in chunks: for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py new file mode 100644 index 00000000..9a98a7ee --- /dev/null +++ b/test/scripts/stream_modify.py @@ -0,0 +1,7 @@ +def modify(chunks): + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo", "bar"), suffix + + +def responseheaders(context, flow): + flow.response.stream = modify
\ No newline at end of file diff --git a/test/test_server.py b/test/test_server.py index e387293f..26770f29 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -266,6 +266,12 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): assert self.master.state.view[-1].response.content == CONTENT_MISSING self.master.set_stream_large_bodies(None) + def test_stream_modify(self): + self.master.load_script(tutils.test_data.path("scripts/stream_modify.py")) + d = self.pathod('200:b"foo"') + assert d.content == "bar" + self.master.unload_scripts() + class TestHTTPAuth(tservers.HTTPProxTest): authenticator = http_auth.BasicProxyAuth(http_auth.PassManSingleUser("test", "test"), "realm") def test_auth(self): |