diff options
Diffstat (limited to 'docs')
35 files changed, 543 insertions, 236 deletions
diff --git a/docs/.gitignore b/docs/.gitignore index 0eba9c59..64b6fff6 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,4 @@ +generated/ src/public node_modules public diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..cc06f081 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,22 @@ +# Mitmproxy Documentation + +This directory houses the mitmproxy documentation available at <https://docs.mitmproxy.org/>. + +## Quick Start + + 1. Install [hugo](https://gohugo.io/). + 2. Windows users: Depending on your git settings, you may need to manually create a symlink from + /docs/src/examples to /examples. + + +Now you can run `hugo server -D` in ./src. + + +## Extended Install + +This is required to modify CSS files. + + 1. Install node, yarn, and [modd](https://github.com/cortesi/modd). + 2. Run `yarn` in this directory to get node-sass. + +You can now run `modd` in this directory instead of running hugo directly. diff --git a/docs/bucketassets/error.html b/docs/bucketassets/error.html new file mode 100644 index 00000000..a855e72b --- /dev/null +++ b/docs/bucketassets/error.html @@ -0,0 +1,9 @@ +Not found +<html> + <head> + <meta http-equiv="refresh" content="0;URL='/stable'" /> + </head> + <body> + Not found - redirecting you to <a href="/stable">latest stable docs</a>. + </body> +</html> diff --git a/docs/bucketassets/robots.txt b/docs/bucketassets/robots.txt new file mode 100644 index 00000000..da4c47d6 --- /dev/null +++ b/docs/bucketassets/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /archive/ +Disallow: /master/
\ No newline at end of file diff --git a/docs/build-archive b/docs/build-archive new file mode 100755 index 00000000..bd11d86e --- /dev/null +++ b/docs/build-archive @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +cd src +DOCS_ARCHIVE=true hugo diff --git a/docs/build-current b/docs/build-current new file mode 100755 index 00000000..7164de6d --- /dev/null +++ b/docs/build-current @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +for script in scripts/* ; do + echo "Generating output for $script ..." + output="${script##*/}" + "$script" > "src/generated/${output%.*}.html" +done + +cd src +hugo diff --git a/docs/ci b/docs/ci new file mode 100755 index 00000000..ab442257 --- /dev/null +++ b/docs/ci @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# This script gets run from CI to render and upload docs + +./build-current + +# Only upload if we have defined credentials - we only have these defined for +# trusted commits (i.e. not PRs). +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then + aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master + aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/master/*" +fi diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py index e61733a0..05cc7a0f 100755 --- a/docs/scripts/filters.py +++ b/docs/scripts/filters.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from mitmproxy import flowfilter @@ -6,4 +6,4 @@ from mitmproxy import flowfilter print("<table class=\"table filtertable\"><tbody>") for i in flowfilter.help: print("<tr><th>%s</th><td>%s</td></tr>" % i) -print("</tbody></table>")
\ No newline at end of file +print("</tbody></table>") diff --git a/docs/scripts/options.py b/docs/scripts/options.py new file mode 100755 index 00000000..ff7d0f7f --- /dev/null +++ b/docs/scripts/options.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from mitmproxy import options, optmanager +from mitmproxy.tools import dump, console, web + +masters = { + "mitmproxy": console.master.ConsoleMaster, + "mitmdump": dump.DumpMaster, + "mitmweb": web.master.WebMaster +} + +unified_options = {} + +for tool_name, master in masters.items(): + opts = options.Options() + inst = master(opts) + for key, option in optmanager.dump_dicts(opts).items(): + if key in unified_options: + unified_options[key]['tools'].append(tool_name) + else: + unified_options[key] = option + unified_options[key]['tools'] = [tool_name] + +print(""" + <table class=\"table optiontable\"> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + </tr> + </thead> + <tbody> + """.strip()) +for key, option in sorted(unified_options.items(), key=lambda t: t[0]): + print(""" + <tr> + <th>{}<br/>{}</th> + <td>{}</td> + <td>{}<br/> + Default: {} + {} + </td> + </tr> + """.strip().format( + key, + ' '.join(["<span class='badge'>{}</span>".format(t) for t in option['tools']]), + option['type'], + option['help'], + option['default'], + "<br/>Choices: {}".format(', '.join(option['choices'])) if option['choices'] else "", + )) +print("</tbody></table>") diff --git a/docs/setup b/docs/setup new file mode 100755 index 00000000..cb63841a --- /dev/null +++ b/docs/setup @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/error.html s3://docs.mitmproxy.org/error.html +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/robots.txt s3://docs.mitmproxy.org/robots.txt diff --git a/docs/src/config.toml b/docs/src/config.toml index c9cecd8b..ee2b9224 100644 --- a/docs/src/config.toml +++ b/docs/src/config.toml @@ -4,6 +4,7 @@ title = "mitmproxy.org docs" theme = "mitmproxydocs" publishDir = "../public" RelativeURLs = true +googleAnalytics = "UA-4150636" [indexes] tag = "tags" diff --git a/docs/src/content/_index.md b/docs/src/content/_index.md index a977e2db..44d41611 100644 --- a/docs/src/content/_index.md +++ b/docs/src/content/_index.md @@ -1,5 +1,6 @@ --- title: "Introduction" +layout: single menu: overview: weight: 1 diff --git a/docs/src/content/addons-overview.md b/docs/src/content/addons-overview.md index 6aa1207e..fea5feb2 100644 --- a/docs/src/content/addons-overview.md +++ b/docs/src/content/addons-overview.md @@ -35,7 +35,7 @@ Here, for example, is a command that shows the API documentation for the mitmproxy's HTTP flow classes: {{< highlight bash >}} -pydoc mimtproxy.http +pydoc mitmproxy.http {{< /highlight >}} You will be referring to the mitmproxy API documentation frequently, so keep @@ -71,4 +71,4 @@ Here are a few things to note about the code above: objects that are commonly used in addons. We could pass a `ctx` object as the first parameter to every event, but we've found it neater to just expose it as an importable global. In this case, we're using the `ctx.log` object to do our - logging.
\ No newline at end of file + logging. diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 6956ff3f..e6586576 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -19,7 +19,7 @@ configure your target device with the correct proxy settings. Now start a browser on the device, and visit the magic domain **mitm.it**. You should see something like this: -{{< figure src="/certinstall-webapp.png" >}} +{{< figure src="/certinstall-webapp.png" class="has-border" >}} Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. @@ -32,8 +32,8 @@ reason. Below is a list of pointers to manual certificate installation documentation for some common platforms. The mitmproxy CA cert is located in `~/.mitmproxy` after it has been generated at the first start of mitmproxy. -- [IOS](http://jasdev.me/intercepting-ios-traffic) On - iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy +- [IOS](http://jasdev.me/intercepting-ios-traffic) + On iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy root certificate: 1. Go to Settings > General > About > Certificate Trust Settings. 2. Under "Enable full trust for root certificates", turn on trust for @@ -42,13 +42,13 @@ documentation for some common platforms. The mitmproxy CA cert is located in - [Java](https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html) - [Android/Android Simulator](http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets) - [Windows](https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7) -- [Windows (automated)](https://technet.microsoft.com/en-us/library/cc732443.aspx) +- [Windows (automated)](https://technet.microsoft.com/en-us/library/cc732443.aspx) {{< highlight bash >}} certutil.exe -importpfx Root mitmproxy-ca-cert.p12 {{< / highlight >}} - -- [Mac OS X](https://support.apple.com/kb/PH7297?locale=en_US) + +- [Mac OS X](https://support.apple.com/kb/PH20129) - [Ubuntu/Debian]( https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861) - [Mozilla Firefox](https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox) - [Chrome on Linux](https://stackoverflow.com/a/15076602/198996) @@ -90,7 +90,7 @@ The files created by mitmproxy in the .mitmproxy directory are as follows: | mitmproxy-ca-cert.p12 | The certificate in PKCS12 format. For use on Windows. | | mitmproxy-ca-cert.cer | Same file as .pem, but with an extension expected by some Android devices. | -## Using a custom certificate +## Using a custom server certificate You can use your own (leaf) certificate by passing the `--cert [domain=]path_to_certificate` option to mitmproxy. Mitmproxy then uses the @@ -156,7 +156,7 @@ hostname, while using a filename allows a single specific certificate to be used for all SSL connections. Certificate files must be in the PEM format and should contain both the unencrypted private key and the certificate. -### Multiple certs by Hostname +### Multiple client certificates You can specify a directory to `--client-certs`, in which case the matching certificate is looked up by filename. So, if you visit example.org, mitmproxy diff --git a/docs/src/content/concepts-filters.md b/docs/src/content/concepts-filters.md index ada24e32..f0b9a4b2 100644 --- a/docs/src/content/concepts-filters.md +++ b/docs/src/content/concepts-filters.md @@ -10,37 +10,7 @@ menu: Many commands in the mitmproxy tool make use of filter expressions. Filter expressions consist of the following operators: - -<table class="table filtertable"><tbody> -<tr><th>~a</th><td>Match asset in response: CSS, Javascript, Flash, images.</td></tr> -<tr><th>~b regex</th><td>Body</td></tr> -<tr><th>~bq regex</th><td>Request body</td></tr> -<tr><th>~bs regex</th><td>Response body</td></tr> -<tr><th>~c int</th><td>HTTP response code</td></tr> -<tr><th>~d regex</th><td>Domain</td></tr> -<tr><th>~dst regex</th><td>Match destination address</td></tr> -<tr><th>~e</th><td>Match error</td></tr> -<tr><th>~h regex</th><td>Header</td></tr> -<tr><th>~hq regex</th><td>Request header</td></tr> -<tr><th>~hs regex</th><td>Response header</td></tr> -<tr><th>~http</th><td>Match HTTP flows</td></tr> -<tr><th>~m regex</th><td>Method</td></tr> -<tr><th>~marked</th><td>Match marked flows</td></tr> -<tr><th>~q</th><td>Match request with no response</td></tr> -<tr><th>~s</th><td>Match response</td></tr> -<tr><th>~src regex</th><td>Match source address</td></tr> -<tr><th>~t regex</th><td>Content-type header</td></tr> -<tr><th>~tcp</th><td>Match TCP flows</td></tr> -<tr><th>~tq regex</th><td>Request Content-Type header</td></tr> -<tr><th>~ts regex</th><td>Response Content-Type header</td></tr> -<tr><th>~u regex</th><td>URL</td></tr> -<tr><th>~websocket</th><td>Match WebSocket flows</td></tr> -<tr><th>!</th><td>unary not</td></tr> -<tr><th>&</th><td>and</td></tr> -<tr><th>|</th><td>or</td></tr> -<tr><th>(...)</th><td>grouping</td></tr> -</tbody></table> - +{{< readfile file="/generated/filters.html" >}} - Regexes are Python-style - Regexes can be specified as quoted strings diff --git a/docs/src/content/concepts-options.md b/docs/src/content/concepts-options.md index 18776841..b533c175 100644 --- a/docs/src/content/concepts-options.md +++ b/docs/src/content/concepts-options.md @@ -34,33 +34,14 @@ interactively have immediate effect in the running instance, and can be made persistent by saving the settings out to a YAML configuration file (please see the specific tool's interactive help for details on how to do this). -For all tools, options can be set directly by name using the `--set` command-line -option. Please see the command-line help (`--help`) for usage. +For all tools, options can be set directly by name using the `--set` +command-line option. Please see the command-line help (`--help`) for usage. -## Example - -Here is an excerpt showing the first few lines of the complete `--options` -output: - -{{< highlight yaml >}} - -# Add all certificates of the upstream server to the certificate chain -# that will be served to the proxy client, as extras. Type bool. -add_upstream_certs_to_client_chain: false - -# Allow remote clients to connect to proxy. If set to false, client will -# not be able to connect to proxy unless it is on the same network or -# the proxyauth option is set Type bool. -allow_remote: false - -# Strip out request headers that might cause the server to return -# 304-not-modified. Type bool. -anticache: false - -# Try to convince servers to send us un-compressed data. Type bool. -anticomp: false - -{{< /highlight >}} +## Available Options +This list might not reflect what is actually available in your current mitmproxy +environment. For an up-to-date list please use the `--options` flag for each of +the mitmproxy tools. +{{< readfile file="/generated/options.html" >}} diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md new file mode 100644 index 00000000..fc056545 --- /dev/null +++ b/docs/src/content/concepts-protocols.md @@ -0,0 +1,93 @@ +--- +title: "Protocols" +menu: + concepts: + weight: 7 +--- + +# Protocols + +## HTTP/1.0 and HTTP/1.1 + +[RFC7230: HTTP/1.1: Message Syntax and Routing](http://tools.ietf.org/html/rfc7230) + +[RFC7231: HTTP/1.1: Semantics and Content](http://tools.ietf.org/html/rfc7231) + +HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, +which takes care of all semantics and on-the-wire parsing/serialization tasks. + +mitmproxy currently does not support HTTP trailers - but if you want to send +us a PR, we promise to take look! + +## HTTP/2 + +[RFC7540: Hypertext Transfer Protocol Version 2 (HTTP/2)](http://tools.ietf.org/html/rfc7540>) + +HTTP/2 support in mitmproxy is based on +[hyper-h2](https://github.com/python-hyper/hyper-h2). It fully encapsulates the +internal state of HTTP/2 connections and provides an easy-to-use event-based +API. mitmproxy supports the majority of HTTP/2 feature and tries to +transparently pass-through as much information as possible. + +mitmproxy currently does not support HTTP/2 trailers - but if you want to send +us a PR, we promise to take look! + +mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the +major browser vendors have implemented it. + +Some websites are still having problems with correct HTTP/2 support in their +webservers and can cause errors, dropped connectiones, or simply no response at +all. We are trying to be as tolerant and forgiving as possible with the types of +data we send and receive, but +[some](https://github.com/mitmproxy/mitmproxy/issues/1745) +[faulty](https://github.com/mitmproxy/mitmproxy/issues/2823) +[implementations](https://github.com/mitmproxy/mitmproxy/issues/1824) +[simply](https://github.com/mitmproxy/mitmproxy/issues/1891) don't work well +with mitmproxy. + +In order to increase the compatibility of mitmproxy with HTTP/2 webservers, we +default to NOT forward any priority information that is sent by a client. You +can enable it with: `http2_priority=true`. + +## WebSocket + +[RFC6455: The WebSocket Protocol](http://tools.ietf.org/html/rfc6455) + +[RFC7692: Compression Extensions for WebSocket](http://tools.ietf.org/html/rfc7692) + +WebSocket support in mitmproxy is based on [wsproto] +(https://github.com/python-hyper/wsproto) project. It fully encapsulates +WebSocket frames/messages/connections and provides an easy-to-use event-based +API. + +mitmproxy fully supports the compression extension for WebSocket messages, +provided by wsproto. Message contents are automatically compressed and +decompressed before firing events. + +mitmproxy currently does not display WebSocket messages in the console or web +UI. Only the WebSocket handshake flow is shown, which contains a reference to +the parent flow for all messages exchanged over this connection. + +If an endpoint sends a PING to mitmproxy, a PONG will be sent back immediately +(with the same payload if present). To keep the other connection alive, a new +PING (without a payload) is sent to the other endpoint. Unsolicited PONG's are +not forwarded. All PING's and PONG's are logged (with payload if present). + +## Raw TCP / TCP Proxy / Fallback + +In case mitmproxy does not handle a specific protocol, you can exempt +hostnames from processing, so that mitmproxy acts as a generic TCP forwarder. +This feature is closely related to the *passthrough* functionality, +but differs in two important aspects: + + * The raw TCP messages are printed to the event log. + * SSL connections will be intercepted. + +Please note that message interception or modification are not possible yet. If +you are not interested in the raw TCP messages, you should use the ignore +domains feature. + +| | | +| ------------------ | ------------------ | +| command-line alias | `--tcp HOST` | +| mitmproxy shortcut | press `O` then `T` | diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 3d99e9dc..ee5b9f57 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -27,87 +27,50 @@ At the moment, mitmproxy supports transparent proxying on OSX Lion and above, and all current flavors of Linux. -## Linux fully transparent mode - -By default mitmproxy will use its own local IP address for its server-side -connections. In case this isn't desired, the --spoof-source-address argument can -be used to use the client's IP address for server-side connections. The -following config is required for this mode to work: - -{{< highlight bash >}} -CLIENT_NET=192.168.1.0/24 -TABLE_ID=100 -MARK=1 - -echo "$TABLE_ID mitmproxy" >> /etc/iproute2/rt_tables -iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK -iptables -t nat \ - -A PREROUTING -p tcp -s $CLIENT_NET \ - --match multiport --dports 80,443 -j \ - REDIRECT --to-port 8080 - -ip rule add fwmark $MARK lookup $TABLE_ID -ip route add local $CLIENT_NET dev lo table $TABLE_ID -{{< / highlight >}} - -This mode does require root privileges though. There's a wrapper in the examples -directory called 'mitmproxy_shim.c', which will enable you to use this mode with -dropped privileges. It can be used as follows: - -{{< highlight bash >}} -gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap -sudo chown root:root mitmproxy_shim -sudo chmod u+s mitmproxy_shim -./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address -{{< / highlight >}} - - - ## Linux On Linux, mitmproxy integrates with the iptables redirection mechanism to achieve transparent mode. -### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) - -### 2. Enable IP forwarding: +### 1. Enable IP forwarding. {{< highlight bash >}} sysctl -w net.ipv4.ip_forward=1 sysctl -w net.ipv6.conf.all.forwarding=1 {{< / highlight >}} -You may also want to consider enabling this permanently in `/etc/sysctl.conf` or -newly created `/etc/sysctl.d/mitmproxy.conf`, see -[here](https://superuser.com/a/625852). +This makes sure that your machine forwards packets instead of rejecting them. -### 3. If your target machine is on the same physical network and you configured it to use a custom gateway, disable ICMP redirects: +If you want to persist this across reboots, you need to adjust your `/etc/sysctl.conf` or +a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com/a/625852)). + +### 2. Disable ICMP redirects. {{< highlight bash >}} sysctl -w net.ipv4.conf.all.send_redirects=0 {{< / highlight >}} -You may also want to consider enabling this permanently in `/etc/sysctl.conf` or -a newly created `/etc/sysctl.d/mitmproxy.conf`, see -[here](https://superuser.com/a/625852). +If your test device is on the same physical network, your machine shouldn't inform the device that +there's a shorter route available by skipping the proxy. + +If you want to persist this across reboots, see above. -### 4. Create an iptables ruleset that redirects the desired traffic to the mitmproxy port +### 3. Create an iptables ruleset that redirects the desired traffic to mitmproxy. Details will differ according to your setup, but the ruleset should look something like this: {{< highlight bash >}} - iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 - iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 - ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 - ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 +iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 +iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 +ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 +ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 {{< / highlight >}} - You may also want to consider enabling this permanently with the -`iptables-persistent` package, see -[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html). +If you want to persist this across reboots, you can use the `iptables-persistent` package (see +[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html)). -### 5. Fire up mitmproxy +### 4. Fire up mitmproxy. You probably want a command like this: @@ -118,24 +81,22 @@ mitmproxy --mode transparent --showhost The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 5. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the -default gateway. For a detailed walkthrough, have a look at the [tutorial for -transparently proxying VMs]({{< relref "howto-transparent-vms" >}}). +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). -## OpenBSD -### 1 [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) +## OpenBSD -### 2. Enable IP forwarding +### 1. Enable IP forwarding. {{< highlight bash >}} sudo sysctl -w net.inet.ip.forwarding=1 {{< / highlight >}} -### 3. Place the following two lines in **/etc/pf.conf** +### 2. Place the following two lines in **/etc/pf.conf**. {{< highlight none >}} mitm_if = "re2" @@ -146,19 +107,19 @@ These rules tell pf to divert all traffic from `$mitm_if` destined for port 80 or 443 to the local mitmproxy instance running on port 8080. You should replace `$mitm_if` value with the interface on which your test device will appear. -### 4. Enable the pf ruleset and enable it +### 3. Configure pf with the rules. {{< highlight bash >}} doas pfctl -f /etc/pf.conf {{< / highlight >}} -And now enable it: +### 4. And now enable it. {{< highlight bash >}} doas pfctl -e {{< / highlight >}} -### 5. Fire up mitmproxy +### 5. Fire up mitmproxy. You probably want a command like this: @@ -169,10 +130,11 @@ mitmproxy --mode transparent --showhost The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 6. Finally, configure your test device. + +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). -Set the test device up to use the host on which mitmproxy is running as the -default gateway. {{% note %}} @@ -195,15 +157,13 @@ packet filter from the OpenBSD project, which mitmproxy uses to implement transparent mode on OSX. Note that this means we don't support transparent mode for earlier versions of OSX. -### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) - -### 2. Enable IP forwarding +### 1. Enable IP forwarding. {{< highlight bash >}} sudo sysctl -w net.inet.ip.forwarding=1 {{< / highlight >}} -### 3. Place the following two lines in a file called, say, **pf.conf** +### 2. Place the following two lines in a file called, say, **pf.conf**. {{< highlight none >}} @@ -212,21 +172,21 @@ rdr on en0 inet proto tcp to any port {80, 443} -> 127.0.0.1 port 8080 These rules tell pf to redirect all traffic destined for port 80 or 443 to the local mitmproxy instance running on port 8080. You should replace -`en2` with the interface on which your test device will appear. +`en0` with the interface on which your test device will appear. -### 4. Configure pf with the rules +### 3. Configure pf with the rules. {{< highlight bash >}} sudo pfctl -f pf.conf {{< / highlight >}} -### 5. And now enable it +### 4. And now enable it. {{< highlight bash >}} sudo pfctl -e {{< / highlight >}} -### 6. Configure sudoers to allow mitmproxy to access pfctl +### 5. Configure sudoers to allow mitmproxy to access pfctl. Edit the file **/etc/sudoers** on your system as root. Add the following line to the end of the file: @@ -240,7 +200,7 @@ state` as root without a password. This only allows inspection of the state table, so should not be an undue security risk. If you're special feel free to tighten the restriction up to the user running mitmproxy. -### 7. Fire up mitmproxy +### 6. Fire up mitmproxy. You probably want a command like this: @@ -251,10 +211,10 @@ mitmproxy --mode transparent --showhost The `--mode transparent` flag turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 7. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the -default gateway. +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). {{% note %}} Note that the **rdr** rules in the pf.conf given above only apply to @@ -267,3 +227,38 @@ flexible to cater for a range of creative possibilities, like intercepting traffic emanating from VMs. See the **pf.conf** man page for more. {{% /note %}} + + +## "Full" transparent mode on Linux + +By default mitmproxy will use its own local IP address for its server-side +connections. In case this isn't desired, the --spoof-source-address argument can +be used to use the client's IP address for server-side connections. The +following config is required for this mode to work: + +{{< highlight bash >}} +CLIENT_NET=192.168.1.0/24 +TABLE_ID=100 +MARK=1 + +echo "$TABLE_ID mitmproxy" >> /etc/iproute2/rt_tables +iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK +iptables -t nat \ + -A PREROUTING -p tcp -s $CLIENT_NET \ + --match multiport --dports 80,443 -j \ + REDIRECT --to-port 8080 + +ip rule add fwmark $MARK lookup $TABLE_ID +ip route add local $CLIENT_NET dev lo table $TABLE_ID +{{< / highlight >}} + +This mode does require root privileges though. There's a wrapper in the examples +directory called 'mitmproxy_shim.c', which will enable you to use this mode with +dropped privileges. It can be used as follows: + +{{< highlight bash >}} +gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap +sudo chown root:root mitmproxy_shim +sudo chmod u+s mitmproxy_shim +./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address +{{< / highlight >}} diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md new file mode 100644 index 00000000..588223ac --- /dev/null +++ b/docs/src/content/howto-wireshark-tls.md @@ -0,0 +1,28 @@ +--- +title: "Wireshark and SSL/TLS" +menu: + howto: + weight: 1 +--- + +# Wireshark and SSL/TLS Master Secrets + +The SSL/SSL master keys can be logged by mitmproxy so that external programs can +decrypt SSL/TLS connections both from and to the proxy. Recent versions of +Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. + +Key logging is enabled by setting the environment variable `SSLKEYLOGFILE` so +that it points to a writable text file: +{{< highlight bash >}} +SSLKEYLOGFILE="$PWD/.mitmproxy/sslkeylogfile.txt" mitmproxy +{{< / highlight >}} +You can also `export` this environment variable to make it persistent for all applications started from your current shell session. + +You can specify the key file path in Wireshark via `Edit -> Preferences -> +Protocols -> SSL -> (Pre)-Master-Secret log filename`. If your SSLKEYLOGFILE +does not exist yet, just create an empty text file, so you can select it in +Wireshark (or run mitmproxy to create and collect master secrets). + +Note that `SSLKEYLOGFILE` is respected by other programs as well, e.g., Firefox +and Chrome. If this creates any issues, you can use `MITMPROXY_SSLKEYLOGFILE` +instead without affecting other applications. diff --git a/docs/src/content/overview-installation.md b/docs/src/content/overview-installation.md index 00941cb8..d69805db 100644 --- a/docs/src/content/overview-installation.md +++ b/docs/src/content/overview-installation.md @@ -73,14 +73,14 @@ security considerations apply as for our binary packages. ## Installation on Linux via pip3 -Please make sure to install Python 3.5 (or higher) and pip3 for your +Please make sure to install Python 3.6 (or higher) and pip3 for your distribution. If your distribution does not provide a suitable Python version, you can use [pyenv](https://github.com/yyuu/pyenv) to get a recent Python environment. {{< highlight bash >}} -sudo apt install python3-pip # Debian 8 or higher, Ubuntu 16.04 or higher -sudo dnf install python3-pip # Fedora 24 or higher +sudo apt install python3-pip # Debian 10 or higher, Ubuntu 17.10 or higher +sudo dnf install python3-pip # Fedora 26 or higher sudo pacman -S python-pip # Arch Linux {{< / highlight >}} @@ -98,7 +98,7 @@ sudo pip3 install mitmproxy ## Installation on Windows via pip3 -First, install the latest version of Python 3.5 or higher from the +First, install the latest version of Python 3.6 or higher from the [Python website](https://www.python.org/downloads/windows/). During installation, make sure to select Add Python to PATH. There are no other dependencies on Windows. diff --git a/docs/src/generated/.gitkeep b/docs/src/generated/.gitkeep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/docs/src/generated/.gitkeep diff --git a/docs/src/layouts/_default/single.html b/docs/src/layouts/_default/single.html index 4a8baf53..801b6341 100644 --- a/docs/src/layouts/_default/single.html +++ b/docs/src/layouts/_default/single.html @@ -1,10 +1,12 @@ -{{ partial "header.html" . }} -<div class="columns"> - <div class="column is-one-quarter sidebody"> - {{ partial "sidebar.html" . }} +{{ partial "header" . }} +<div class="columns container is-marginless"> + <div id="sidebar" class="column is-one-quarter"> + {{ partial "sidebar" . }} </div> - <div class="column content mainbody"> - {{.Content}} + <div id="main" class="column content"> + {{ partial "outdated" . }} + {{ partial "edit-on-github" . }} + {{ partial "add-anchors" .Content}} </div> </div> {{ partial "footer.html" . }} diff --git a/docs/src/layouts/index.html b/docs/src/layouts/index.html deleted file mode 100644 index 4a8baf53..00000000 --- a/docs/src/layouts/index.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ partial "header.html" . }} -<div class="columns"> - <div class="column is-one-quarter sidebody"> - {{ partial "sidebar.html" . }} - </div> - <div class="column content mainbody"> - {{.Content}} - </div> -</div> -{{ partial "footer.html" . }} diff --git a/docs/src/layouts/partials/add-anchors.html b/docs/src/layouts/partials/add-anchors.html new file mode 100644 index 00000000..f7050f7f --- /dev/null +++ b/docs/src/layouts/partials/add-anchors.html @@ -0,0 +1 @@ +{{ . | replaceRE "(<h[1-9] id=\"(.+?)\".*?>)(.+?</h[1-9]>)" "${1}<a class=\"anchor\" href=\"#${2}\"># </a>${3}" | safeHTML }} diff --git a/docs/src/layouts/partials/edit-on-github.html b/docs/src/layouts/partials/edit-on-github.html new file mode 100644 index 00000000..d2c3098c --- /dev/null +++ b/docs/src/layouts/partials/edit-on-github.html @@ -0,0 +1,9 @@ +{{ if and .IsPage (not (getenv "DOCS_ARCHIVE")) }} +<a class="button is-small is-outlined is-link is-pulled-right" + target="_blank" + href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}" +> + Edit on GitHub +</a> +{{ end }} + diff --git a/docs/src/layouts/partials/outdated.html b/docs/src/layouts/partials/outdated.html new file mode 100644 index 00000000..5b3dd6ed --- /dev/null +++ b/docs/src/layouts/partials/outdated.html @@ -0,0 +1,9 @@ +{{- if (getenv "DOCS_ARCHIVE") -}} +<article class="message is-warning"> + <div class="message-body"> + You are not viewing the most up to date version of the documentation. + Click <a href="https://docs.mitmproxy.org/stable{{ .Page.URL }}">here</a> + to view the latest version. + </div> +</article> +{{- end -}} diff --git a/docs/src/layouts/partials/sidebar.html b/docs/src/layouts/partials/sidebar.html index ef853fc6..5ea41c12 100644 --- a/docs/src/layouts/partials/sidebar.html +++ b/docs/src/layouts/partials/sidebar.html @@ -1,24 +1,22 @@ -<div class="sidebar"> - <div class="brand"> - <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs"> - </div> - <div class="version"> - <span class="tag is-info is-rounded is-medium">v3.x</span> - </div> - <aside class="menu"> - <p class="menu-label"> Overview </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "overview") }} +<div class="brand"> + <a href="https://mitmproxy.org/"> + <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs"/> + </a> - <p class="menu-label">Core concepts</p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "concepts") }} +</div> +<nav class="menu"> + <p class="menu-label"> Overview </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "overview") }} - <p class="menu-label"> Addon Development </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "addons") }} + <p class="menu-label">Core concepts</p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "concepts") }} - <p class="menu-label"> HOWTOs </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "howto") }} + <p class="menu-label"> Addon Development </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "addons") }} - <p class="menu-label"> Tutorials </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "tutes") }} - </aside> -</div>
\ No newline at end of file + <p class="menu-label"> HOWTOs </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "howto") }} + + <p class="menu-label"> Tutorials </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "tutes") }} +</nav> diff --git a/docs/src/layouts/shortcodes/readfile.html b/docs/src/layouts/shortcodes/readfile.html new file mode 100644 index 00000000..6860b0f1 --- /dev/null +++ b/docs/src/layouts/shortcodes/readfile.html @@ -0,0 +1,6 @@ +{{$file := .Get "file"}} +{{- if eq (.Get "markdown") "true" -}} +{{- $file | readFile | markdownify -}} +{{- else -}} +{{ $file | readFile | safeHTML }} +{{- end -}} diff --git a/docs/src/static/logo-docs.png b/docs/src/static/logo-docs.png Binary files differindex b37dbd85..a46016dd 100644 --- a/docs/src/static/logo-docs.png +++ b/docs/src/static/logo-docs.png diff --git a/docs/src/themes/mitmproxydocs/layouts/partials/footer.html b/docs/src/themes/mitmproxydocs/layouts/partials/footer.html index 308b1d01..dc9ddc85 100644 --- a/docs/src/themes/mitmproxydocs/layouts/partials/footer.html +++ b/docs/src/themes/mitmproxydocs/layouts/partials/footer.html @@ -1,2 +1,3 @@ +{{ template "_internal/google_analytics_async.html" . }} </body> </html> diff --git a/docs/src/themes/mitmproxydocs/static/css/style.css b/docs/src/themes/mitmproxydocs/static/css/style.css index 14823447..d42e307e 100644 --- a/docs/src/themes/mitmproxydocs/static/css/style.css +++ b/docs/src/themes/mitmproxydocs/static/css/style.css @@ -250,6 +250,21 @@ .chroma .gu { color: #75715e; } +.badge { + color: #fff; + background-color: #6c757d; + display: inline-block; + padding: .25em .4em; + font-size: 75%; + font-weight: 1; + line-height: 1; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: 0.25rem; } + .badge:empty { + display: none; } + @keyframes spinAround { from { transform: rotate(0deg); } @@ -6717,10 +6732,17 @@ label.panel-block { background-color: whitesmoke; padding: 3rem 1.5rem 6rem; } -.sidebody { - height: 100vh; - overflow-x: hidden; - overflow-y: scroll; } +#sidebar { + background-color: #eee; + border-right: 1px solid #c1c1c1; + box-shadow: 0 0 20px rgba(50, 50, 50, 0.2) inset; + padding: 1.75rem; } + #sidebar .brand { + padding: 1rem 0; + text-align: center; } + +#main { + padding: 3rem; } .example { margin-bottom: 1em; } @@ -6731,21 +6753,6 @@ label.panel-block { width: 100%; text-align: right; } -.sidebar .version { - padding: 1em; } - -.sidebar .brand { - background-color: #303030; - color: #c0c0c0; - padding: 1em; - top: 0; } - -.sidebar .menu { - padding: 1em; } - -.mainbody { - padding: 3em; } - code { color: #1a9f1a; font-size: 0.875em; @@ -6754,3 +6761,26 @@ code { .content h2 { padding-top: 1em; border-top: 1px solid #c0c0c0; } + +h1 .anchor, h2 .anchor, h3 .anchor, h4 .anchor, h5 .anchor, h6 .anchor { + display: inline-block; + width: 0; + margin-left: -1.5rem; + margin-right: 1.5rem; + transition: all 100ms ease-in-out; + opacity: 0; } + +h1:hover .anchor, h2:hover .anchor, h3:hover .anchor, h4:hover .anchor, h5:hover .anchor, h6:hover .anchor { + opacity: 1; } + +h1:target, h2:target, h3:target, h4:target, h5:target, h6:target { + color: #C93312; } + h1:target .anchor, h2:target .anchor, h3:target .anchor, h4:target .anchor, h5:target .anchor, h6:target .anchor { + opacity: 1; + color: #C93312; } + +.footnotes p { + display: inline; } + +figure.has-border img { + box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25); } diff --git a/docs/style/badge.scss b/docs/style/badge.scss new file mode 100644 index 00000000..8082f6c7 --- /dev/null +++ b/docs/style/badge.scss @@ -0,0 +1,18 @@ +.badge { + color: #fff; + background-color: #6c757d; + display: inline-block; + padding: .25em .4em; + font-size: 75%; + font-weight: 1; + line-height: 1; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: 0.25rem; + + // Empty badges collapse automatically + &:empty { + display: none; + } +} diff --git a/docs/style/style.scss b/docs/style/style.scss index 2c66a4c9..b404dc42 100644 --- a/docs/style/style.scss +++ b/docs/style/style.scss @@ -1,4 +1,5 @@ @import "./syntax"; +@import "./badge"; $primary: #C93312; $warning-invert: #FFFFFF; @@ -10,10 +11,20 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox @import "../node_modules/bulma/sass/components/_all"; @import "../node_modules/bulma/sass/layout/_all"; -.sidebody { - height: 100vh; - overflow-x: hidden; - overflow-y: scroll; +#sidebar { + background-color: #eee; + border-right: 1px solid #c1c1c1; + box-shadow: 0 0 20px rgba(50, 50, 50, .2) inset; + padding: $column-gap + 1rem; + + .brand { + padding: 1rem 0; + text-align: center; + } +} + +#main { + padding: 3rem; } .example { @@ -28,30 +39,10 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox margin-bottom: 1em; } - -.sidebar { - .version { - padding: 1em; - } - .brand { - background-color: #303030; - color: #c0c0c0; - padding: 1em; - top: 0; - } - .menu { - padding: 1em; - } -} - -.mainbody { - padding: 3em; -} - code { - color: #1a9f1a; - font-size: 0.875em; - font-weight: normal; + color: #1a9f1a; + font-size: 0.875em; + font-weight: normal; } .content { @@ -60,3 +51,32 @@ code { border-top: 1px solid #c0c0c0; } } + +h1, h2, h3, h4, h5, h6 { + .anchor { + display: inline-block; + width: 0; + margin-left: -1.5rem; + margin-right: 1.5rem; + transition: all 100ms ease-in-out; + opacity: 0; + } + &:hover .anchor { + opacity: 1; + } + &:target { + color: $primary; + .anchor { + opacity: 1; + color: $primary + } + } +} + +.footnotes p { + display: inline; +} + +figure.has-border img { + box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25); +} diff --git a/docs/upload-archive b/docs/upload-archive new file mode 100755 index 00000000..3aaeb9be --- /dev/null +++ b/docs/upload-archive @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [[ $# -eq 0 ]] ; then + echo "Please supply a version, e.g. 'v3'" + exit 1 +fi + +# This script uploads docs to a specified archive version. + +SPATH="/archive/$1" + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "$SPATH/*" diff --git a/docs/upload-stable b/docs/upload-stable new file mode 100755 index 00000000..5aea7479 --- /dev/null +++ b/docs/upload-stable @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/stable/*" |
