diff options
98 files changed, 1344 insertions, 1035 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 6891f1b3..90364823 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,22 +9,8 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python35" - TOXENV: "py35" - # TODO: ENABLE WHEN AVAILABLE - # - PYTHON: "C:\\Python36" - # TOXENV: "py36" - - SNAPSHOT_HOST: - secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= - SNAPSHOT_PORT: - secure: TiJPtg60/edYTH8RnoBErg== - SNAPSHOT_USER: - secure: 6yBwmO5gv4vAwoFYII8qjQ== - SNAPSHOT_PASS: - secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV - RTOOL_KEY: - secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM + - PYTHON: "C:\\Python36" + TOXENV: "py36" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -36,17 +22,10 @@ test_script: - ps: | $Env:VERSION = $(python -m mitmproxy.version) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" - tox -e rtool -- wheel - tox -e rtool -- bdist - + tox -e cibuild -- build - ps: | - if( - ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - echo "Decrypt license..." - tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml - $ibVersion = "17.12.0" + if($Env:RTOOL_KEY) { + $ibVersion = "18.2.0" $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe" $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe" if (!(Test-Path $ibSetup)) { @@ -58,8 +37,8 @@ test_script: } echo "Install InstallBuilder..." Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait - # Wait until executable exists - no idea why this is necessary. - while (!(Test-Path $ibCli)) { Start-Sleep 0.1 } + echo "Decrypt license..." + tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml echo "Run InstallBuilder..." &$ibCli ` build ` @@ -73,15 +52,7 @@ test_script: } deploy_script: - # we build binaries on every run, but we only upload them for master snapshots or tags. - ps: | - if( - ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts - tox -e rtool -- upload-snapshot --bdist --wheel --installer - } + ps: tox -e cibuild -- upload cache: - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml diff --git a/.travis.yml b/.travis.yml index c22038fd..3e8b4882 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,9 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 BDIST=1 - - python: 3.5 - env: TOXENV=py35 BDIST=1 - dist: precise + env: TOXENV=py36 BDIST=1 - python: 3.6 - env: TOXENV=py36 + env: TOXENV=py36 BDIST=1 WHEEL=1 - python: 3.6 env: TOXENV=individual_coverage - language: node_js @@ -42,6 +39,17 @@ matrix: yarn: true directories: - web/node_modules + - python: 3.6 + env: NAME=docs TOXENV=docs + install: + - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + - sudo dpkg -i hugo*.deb + - pip install tox virtualenv setuptools + - pyenv global system 3.6 + script: + - tox + after_success: + - echo done install: - | @@ -50,12 +58,11 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3 - pyenv global 3.5.3 - pyenv shell 3.5.3 - pip install -U pip setuptools wheel virtualenv + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.5 + pyenv global 3.6.5 + pyenv shell 3.6.5 fi - - pip install tox + - pip install tox virtualenv setuptools script: - tox -- --verbose --cov-report=term @@ -63,16 +70,14 @@ script: if [[ $BDIST == "1" ]] then git fetch --unshallow --tags - tox -e rtool -- bdist + tox -e cibuild -- build fi after_success: - # we build binaries on every run, but we only upload them for master snapshots or tags. - | - if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" ]] then - tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts - tox -e rtool -- upload-snapshot --bdist + tox -e cibuild -- upload fi notifications: @@ -1,6 +1,18 @@ -23 February 2018: mitmproxy 3.0 +05 April 2018: mitmproxy 3.0.4 - * Fix a quote-related issue affecting the mitmproxy console command prompt + * Fix an issue that caused mitmproxy to not retry HTTP requests on timeout. + + * Various other fixes (@kira0204, @fenilgandhi, @tran-tien-dat, @smonami, + @luzpaz, @fristonio, @kajojify, @Oliver-Fish, @hcbarry, @jplochocki, @MikeShi42, + @ghillu, @emilstahl) + +25 February 2018: mitmproxy 3.0.3 + + * Fix an issue that caused mitmproxy to lose keyboard control after spawning an external editor. + +23 February 2018: mitmproxy 3.0.1 + + * Fix a quote-related issue affecting the mitmproxy console command prompt. 22 February 2018: mitmproxy 3.0 @@ -26,9 +26,9 @@ and pathod websites. |mitmproxy_site| -The latest documentation for mitmproxy is available on our website. +The documentation for mitmproxy is available on our website: -|mitmproxy_docs| +|mitmproxy_docs_stable| |mitmproxy_docs_master| Join our discussion forum on Discourse to ask questions, help @@ -45,7 +45,7 @@ Join our developer chat on Slack if you would like to contribute to mitmproxy it Installation ------------ -The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__. +The installation instructions are `here <https://docs.mitmproxy.org/stable/overview-installation>`__. If you want to contribute changes, keep on reading. Contributing @@ -150,9 +150,13 @@ with the following command: :target: https://mitmproxy.org/ :alt: mitmproxy.org -.. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg - :target: https://mitmproxy.org/docs/latest/ - :alt: mitmproxy documentation +.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg + :target: https://docs.mitmproxy.org/stable/ + :alt: mitmproxy documentation stable + +.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg + :target: https://docs.mitmproxy.org/master/ + :alt: mitmproxy documentation master .. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg :target: https://discourse.mitmproxy.org @@ -182,7 +186,7 @@ with the following command: :target: https://pypi.python.org/pypi/mitmproxy :alt: Supported Python versions -.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation +.. _`advanced installation`: https://docs.mitmproxy.org/stable/overview-installation/#advanced-installation .. _virtualenv: https://virtualenv.pypa.io/ .. _`pytest`: http://pytest.org/ .. _tox: https://tox.readthedocs.io/ @@ -1,8 +1,8 @@ $ErrorActionPreference = "Stop" $pyver = python --version -if($pyver -notmatch "3\.[5-9]") { - Write-Warning "Unexpected Python version, expected Python 3.5 or above: $pyver" +if($pyver -notmatch "3\.[6-9]") { + Write-Warning "Unexpected Python version, expected Python 3.6 or above: $pyver" } python -m venv .\venv --copies @@ -17,4 +17,4 @@ echo @" * Installed all dependencies into the virtualenv. * Activated virtualenv environment. -"@
\ No newline at end of file +"@ diff --git a/docs/.gitignore b/docs/.gitignore index 0eba9c59..64b6fff6 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,4 @@ +generated/ src/public node_modules public diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..cc06f081 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,22 @@ +# Mitmproxy Documentation + +This directory houses the mitmproxy documentation available at <https://docs.mitmproxy.org/>. + +## Quick Start + + 1. Install [hugo](https://gohugo.io/). + 2. Windows users: Depending on your git settings, you may need to manually create a symlink from + /docs/src/examples to /examples. + + +Now you can run `hugo server -D` in ./src. + + +## Extended Install + +This is required to modify CSS files. + + 1. Install node, yarn, and [modd](https://github.com/cortesi/modd). + 2. Run `yarn` in this directory to get node-sass. + +You can now run `modd` in this directory instead of running hugo directly. diff --git a/docs/bucketassets/error.html b/docs/bucketassets/error.html new file mode 100644 index 00000000..a855e72b --- /dev/null +++ b/docs/bucketassets/error.html @@ -0,0 +1,9 @@ +Not found +<html> + <head> + <meta http-equiv="refresh" content="0;URL='/stable'" /> + </head> + <body> + Not found - redirecting you to <a href="/stable">latest stable docs</a>. + </body> +</html> diff --git a/docs/bucketassets/robots.txt b/docs/bucketassets/robots.txt new file mode 100644 index 00000000..da4c47d6 --- /dev/null +++ b/docs/bucketassets/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /archive/ +Disallow: /master/
\ No newline at end of file diff --git a/docs/build-archive b/docs/build-archive new file mode 100755 index 00000000..bd11d86e --- /dev/null +++ b/docs/build-archive @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +cd src +DOCS_ARCHIVE=true hugo diff --git a/docs/build-current b/docs/build-current new file mode 100755 index 00000000..7164de6d --- /dev/null +++ b/docs/build-current @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +for script in scripts/* ; do + echo "Generating output for $script ..." + output="${script##*/}" + "$script" > "src/generated/${output%.*}.html" +done + +cd src +hugo diff --git a/docs/ci b/docs/ci new file mode 100755 index 00000000..ab442257 --- /dev/null +++ b/docs/ci @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# This script gets run from CI to render and upload docs + +./build-current + +# Only upload if we have defined credentials - we only have these defined for +# trusted commits (i.e. not PRs). +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then + aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master + aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/master/*" +fi diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py index e61733a0..05cc7a0f 100755 --- a/docs/scripts/filters.py +++ b/docs/scripts/filters.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from mitmproxy import flowfilter @@ -6,4 +6,4 @@ from mitmproxy import flowfilter print("<table class=\"table filtertable\"><tbody>") for i in flowfilter.help: print("<tr><th>%s</th><td>%s</td></tr>" % i) -print("</tbody></table>")
\ No newline at end of file +print("</tbody></table>") diff --git a/docs/scripts/options.py b/docs/scripts/options.py new file mode 100755 index 00000000..ff7d0f7f --- /dev/null +++ b/docs/scripts/options.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from mitmproxy import options, optmanager +from mitmproxy.tools import dump, console, web + +masters = { + "mitmproxy": console.master.ConsoleMaster, + "mitmdump": dump.DumpMaster, + "mitmweb": web.master.WebMaster +} + +unified_options = {} + +for tool_name, master in masters.items(): + opts = options.Options() + inst = master(opts) + for key, option in optmanager.dump_dicts(opts).items(): + if key in unified_options: + unified_options[key]['tools'].append(tool_name) + else: + unified_options[key] = option + unified_options[key]['tools'] = [tool_name] + +print(""" + <table class=\"table optiontable\"> + <thead> + <tr> + <th>Name</th> + <th>Type</th> + <th>Description</th> + </tr> + </thead> + <tbody> + """.strip()) +for key, option in sorted(unified_options.items(), key=lambda t: t[0]): + print(""" + <tr> + <th>{}<br/>{}</th> + <td>{}</td> + <td>{}<br/> + Default: {} + {} + </td> + </tr> + """.strip().format( + key, + ' '.join(["<span class='badge'>{}</span>".format(t) for t in option['tools']]), + option['type'], + option['help'], + option['default'], + "<br/>Choices: {}".format(', '.join(option['choices'])) if option['choices'] else "", + )) +print("</tbody></table>") diff --git a/docs/setup b/docs/setup new file mode 100755 index 00000000..cb63841a --- /dev/null +++ b/docs/setup @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/error.html s3://docs.mitmproxy.org/error.html +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/robots.txt s3://docs.mitmproxy.org/robots.txt diff --git a/docs/src/config.toml b/docs/src/config.toml index c9cecd8b..ee2b9224 100644 --- a/docs/src/config.toml +++ b/docs/src/config.toml @@ -4,6 +4,7 @@ title = "mitmproxy.org docs" theme = "mitmproxydocs" publishDir = "../public" RelativeURLs = true +googleAnalytics = "UA-4150636" [indexes] tag = "tags" diff --git a/docs/src/content/_index.md b/docs/src/content/_index.md index a977e2db..44d41611 100644 --- a/docs/src/content/_index.md +++ b/docs/src/content/_index.md @@ -1,5 +1,6 @@ --- title: "Introduction" +layout: single menu: overview: weight: 1 diff --git a/docs/src/content/addons-overview.md b/docs/src/content/addons-overview.md index 6aa1207e..fea5feb2 100644 --- a/docs/src/content/addons-overview.md +++ b/docs/src/content/addons-overview.md @@ -35,7 +35,7 @@ Here, for example, is a command that shows the API documentation for the mitmproxy's HTTP flow classes: {{< highlight bash >}} -pydoc mimtproxy.http +pydoc mitmproxy.http {{< /highlight >}} You will be referring to the mitmproxy API documentation frequently, so keep @@ -71,4 +71,4 @@ Here are a few things to note about the code above: objects that are commonly used in addons. We could pass a `ctx` object as the first parameter to every event, but we've found it neater to just expose it as an importable global. In this case, we're using the `ctx.log` object to do our - logging.
\ No newline at end of file + logging. diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 6956ff3f..e6586576 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -19,7 +19,7 @@ configure your target device with the correct proxy settings. Now start a browser on the device, and visit the magic domain **mitm.it**. You should see something like this: -{{< figure src="/certinstall-webapp.png" >}} +{{< figure src="/certinstall-webapp.png" class="has-border" >}} Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. @@ -32,8 +32,8 @@ reason. Below is a list of pointers to manual certificate installation documentation for some common platforms. The mitmproxy CA cert is located in `~/.mitmproxy` after it has been generated at the first start of mitmproxy. -- [IOS](http://jasdev.me/intercepting-ios-traffic) On - iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy +- [IOS](http://jasdev.me/intercepting-ios-traffic) + On iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy root certificate: 1. Go to Settings > General > About > Certificate Trust Settings. 2. Under "Enable full trust for root certificates", turn on trust for @@ -42,13 +42,13 @@ documentation for some common platforms. The mitmproxy CA cert is located in - [Java](https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html) - [Android/Android Simulator](http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets) - [Windows](https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7) -- [Windows (automated)](https://technet.microsoft.com/en-us/library/cc732443.aspx) +- [Windows (automated)](https://technet.microsoft.com/en-us/library/cc732443.aspx) {{< highlight bash >}} certutil.exe -importpfx Root mitmproxy-ca-cert.p12 {{< / highlight >}} - -- [Mac OS X](https://support.apple.com/kb/PH7297?locale=en_US) + +- [Mac OS X](https://support.apple.com/kb/PH20129) - [Ubuntu/Debian]( https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861) - [Mozilla Firefox](https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox) - [Chrome on Linux](https://stackoverflow.com/a/15076602/198996) @@ -90,7 +90,7 @@ The files created by mitmproxy in the .mitmproxy directory are as follows: | mitmproxy-ca-cert.p12 | The certificate in PKCS12 format. For use on Windows. | | mitmproxy-ca-cert.cer | Same file as .pem, but with an extension expected by some Android devices. | -## Using a custom certificate +## Using a custom server certificate You can use your own (leaf) certificate by passing the `--cert [domain=]path_to_certificate` option to mitmproxy. Mitmproxy then uses the @@ -156,7 +156,7 @@ hostname, while using a filename allows a single specific certificate to be used for all SSL connections. Certificate files must be in the PEM format and should contain both the unencrypted private key and the certificate. -### Multiple certs by Hostname +### Multiple client certificates You can specify a directory to `--client-certs`, in which case the matching certificate is looked up by filename. So, if you visit example.org, mitmproxy diff --git a/docs/src/content/concepts-filters.md b/docs/src/content/concepts-filters.md index ada24e32..f0b9a4b2 100644 --- a/docs/src/content/concepts-filters.md +++ b/docs/src/content/concepts-filters.md @@ -10,37 +10,7 @@ menu: Many commands in the mitmproxy tool make use of filter expressions. Filter expressions consist of the following operators: - -<table class="table filtertable"><tbody> -<tr><th>~a</th><td>Match asset in response: CSS, Javascript, Flash, images.</td></tr> -<tr><th>~b regex</th><td>Body</td></tr> -<tr><th>~bq regex</th><td>Request body</td></tr> -<tr><th>~bs regex</th><td>Response body</td></tr> -<tr><th>~c int</th><td>HTTP response code</td></tr> -<tr><th>~d regex</th><td>Domain</td></tr> -<tr><th>~dst regex</th><td>Match destination address</td></tr> -<tr><th>~e</th><td>Match error</td></tr> -<tr><th>~h regex</th><td>Header</td></tr> -<tr><th>~hq regex</th><td>Request header</td></tr> -<tr><th>~hs regex</th><td>Response header</td></tr> -<tr><th>~http</th><td>Match HTTP flows</td></tr> -<tr><th>~m regex</th><td>Method</td></tr> -<tr><th>~marked</th><td>Match marked flows</td></tr> -<tr><th>~q</th><td>Match request with no response</td></tr> -<tr><th>~s</th><td>Match response</td></tr> -<tr><th>~src regex</th><td>Match source address</td></tr> -<tr><th>~t regex</th><td>Content-type header</td></tr> -<tr><th>~tcp</th><td>Match TCP flows</td></tr> -<tr><th>~tq regex</th><td>Request Content-Type header</td></tr> -<tr><th>~ts regex</th><td>Response Content-Type header</td></tr> -<tr><th>~u regex</th><td>URL</td></tr> -<tr><th>~websocket</th><td>Match WebSocket flows</td></tr> -<tr><th>!</th><td>unary not</td></tr> -<tr><th>&</th><td>and</td></tr> -<tr><th>|</th><td>or</td></tr> -<tr><th>(...)</th><td>grouping</td></tr> -</tbody></table> - +{{< readfile file="/generated/filters.html" >}} - Regexes are Python-style - Regexes can be specified as quoted strings diff --git a/docs/src/content/concepts-options.md b/docs/src/content/concepts-options.md index 18776841..b533c175 100644 --- a/docs/src/content/concepts-options.md +++ b/docs/src/content/concepts-options.md @@ -34,33 +34,14 @@ interactively have immediate effect in the running instance, and can be made persistent by saving the settings out to a YAML configuration file (please see the specific tool's interactive help for details on how to do this). -For all tools, options can be set directly by name using the `--set` command-line -option. Please see the command-line help (`--help`) for usage. +For all tools, options can be set directly by name using the `--set` +command-line option. Please see the command-line help (`--help`) for usage. -## Example - -Here is an excerpt showing the first few lines of the complete `--options` -output: - -{{< highlight yaml >}} - -# Add all certificates of the upstream server to the certificate chain -# that will be served to the proxy client, as extras. Type bool. -add_upstream_certs_to_client_chain: false - -# Allow remote clients to connect to proxy. If set to false, client will -# not be able to connect to proxy unless it is on the same network or -# the proxyauth option is set Type bool. -allow_remote: false - -# Strip out request headers that might cause the server to return -# 304-not-modified. Type bool. -anticache: false - -# Try to convince servers to send us un-compressed data. Type bool. -anticomp: false - -{{< /highlight >}} +## Available Options +This list might not reflect what is actually available in your current mitmproxy +environment. For an up-to-date list please use the `--options` flag for each of +the mitmproxy tools. +{{< readfile file="/generated/options.html" >}} diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md new file mode 100644 index 00000000..fc056545 --- /dev/null +++ b/docs/src/content/concepts-protocols.md @@ -0,0 +1,93 @@ +--- +title: "Protocols" +menu: + concepts: + weight: 7 +--- + +# Protocols + +## HTTP/1.0 and HTTP/1.1 + +[RFC7230: HTTP/1.1: Message Syntax and Routing](http://tools.ietf.org/html/rfc7230) + +[RFC7231: HTTP/1.1: Semantics and Content](http://tools.ietf.org/html/rfc7231) + +HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, +which takes care of all semantics and on-the-wire parsing/serialization tasks. + +mitmproxy currently does not support HTTP trailers - but if you want to send +us a PR, we promise to take look! + +## HTTP/2 + +[RFC7540: Hypertext Transfer Protocol Version 2 (HTTP/2)](http://tools.ietf.org/html/rfc7540>) + +HTTP/2 support in mitmproxy is based on +[hyper-h2](https://github.com/python-hyper/hyper-h2). It fully encapsulates the +internal state of HTTP/2 connections and provides an easy-to-use event-based +API. mitmproxy supports the majority of HTTP/2 feature and tries to +transparently pass-through as much information as possible. + +mitmproxy currently does not support HTTP/2 trailers - but if you want to send +us a PR, we promise to take look! + +mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the +major browser vendors have implemented it. + +Some websites are still having problems with correct HTTP/2 support in their +webservers and can cause errors, dropped connectiones, or simply no response at +all. We are trying to be as tolerant and forgiving as possible with the types of +data we send and receive, but +[some](https://github.com/mitmproxy/mitmproxy/issues/1745) +[faulty](https://github.com/mitmproxy/mitmproxy/issues/2823) +[implementations](https://github.com/mitmproxy/mitmproxy/issues/1824) +[simply](https://github.com/mitmproxy/mitmproxy/issues/1891) don't work well +with mitmproxy. + +In order to increase the compatibility of mitmproxy with HTTP/2 webservers, we +default to NOT forward any priority information that is sent by a client. You +can enable it with: `http2_priority=true`. + +## WebSocket + +[RFC6455: The WebSocket Protocol](http://tools.ietf.org/html/rfc6455) + +[RFC7692: Compression Extensions for WebSocket](http://tools.ietf.org/html/rfc7692) + +WebSocket support in mitmproxy is based on [wsproto] +(https://github.com/python-hyper/wsproto) project. It fully encapsulates +WebSocket frames/messages/connections and provides an easy-to-use event-based +API. + +mitmproxy fully supports the compression extension for WebSocket messages, +provided by wsproto. Message contents are automatically compressed and +decompressed before firing events. + +mitmproxy currently does not display WebSocket messages in the console or web +UI. Only the WebSocket handshake flow is shown, which contains a reference to +the parent flow for all messages exchanged over this connection. + +If an endpoint sends a PING to mitmproxy, a PONG will be sent back immediately +(with the same payload if present). To keep the other connection alive, a new +PING (without a payload) is sent to the other endpoint. Unsolicited PONG's are +not forwarded. All PING's and PONG's are logged (with payload if present). + +## Raw TCP / TCP Proxy / Fallback + +In case mitmproxy does not handle a specific protocol, you can exempt +hostnames from processing, so that mitmproxy acts as a generic TCP forwarder. +This feature is closely related to the *passthrough* functionality, +but differs in two important aspects: + + * The raw TCP messages are printed to the event log. + * SSL connections will be intercepted. + +Please note that message interception or modification are not possible yet. If +you are not interested in the raw TCP messages, you should use the ignore +domains feature. + +| | | +| ------------------ | ------------------ | +| command-line alias | `--tcp HOST` | +| mitmproxy shortcut | press `O` then `T` | diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 3d99e9dc..ee5b9f57 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -27,87 +27,50 @@ At the moment, mitmproxy supports transparent proxying on OSX Lion and above, and all current flavors of Linux. -## Linux fully transparent mode - -By default mitmproxy will use its own local IP address for its server-side -connections. In case this isn't desired, the --spoof-source-address argument can -be used to use the client's IP address for server-side connections. The -following config is required for this mode to work: - -{{< highlight bash >}} -CLIENT_NET=192.168.1.0/24 -TABLE_ID=100 -MARK=1 - -echo "$TABLE_ID mitmproxy" >> /etc/iproute2/rt_tables -iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK -iptables -t nat \ - -A PREROUTING -p tcp -s $CLIENT_NET \ - --match multiport --dports 80,443 -j \ - REDIRECT --to-port 8080 - -ip rule add fwmark $MARK lookup $TABLE_ID -ip route add local $CLIENT_NET dev lo table $TABLE_ID -{{< / highlight >}} - -This mode does require root privileges though. There's a wrapper in the examples -directory called 'mitmproxy_shim.c', which will enable you to use this mode with -dropped privileges. It can be used as follows: - -{{< highlight bash >}} -gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap -sudo chown root:root mitmproxy_shim -sudo chmod u+s mitmproxy_shim -./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address -{{< / highlight >}} - - - ## Linux On Linux, mitmproxy integrates with the iptables redirection mechanism to achieve transparent mode. -### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) - -### 2. Enable IP forwarding: +### 1. Enable IP forwarding. {{< highlight bash >}} sysctl -w net.ipv4.ip_forward=1 sysctl -w net.ipv6.conf.all.forwarding=1 {{< / highlight >}} -You may also want to consider enabling this permanently in `/etc/sysctl.conf` or -newly created `/etc/sysctl.d/mitmproxy.conf`, see -[here](https://superuser.com/a/625852). +This makes sure that your machine forwards packets instead of rejecting them. -### 3. If your target machine is on the same physical network and you configured it to use a custom gateway, disable ICMP redirects: +If you want to persist this across reboots, you need to adjust your `/etc/sysctl.conf` or +a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com/a/625852)). + +### 2. Disable ICMP redirects. {{< highlight bash >}} sysctl -w net.ipv4.conf.all.send_redirects=0 {{< / highlight >}} -You may also want to consider enabling this permanently in `/etc/sysctl.conf` or -a newly created `/etc/sysctl.d/mitmproxy.conf`, see -[here](https://superuser.com/a/625852). +If your test device is on the same physical network, your machine shouldn't inform the device that +there's a shorter route available by skipping the proxy. + +If you want to persist this across reboots, see above. -### 4. Create an iptables ruleset that redirects the desired traffic to the mitmproxy port +### 3. Create an iptables ruleset that redirects the desired traffic to mitmproxy. Details will differ according to your setup, but the ruleset should look something like this: {{< highlight bash >}} - iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 - iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 - ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 - ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 +iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 +iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 +ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080 +ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080 {{< / highlight >}} -Â Â You may also want to consider enabling this permanently with the -`iptables-persistent` package, see -[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html). +If you want to persist this across reboots, you can use the `iptables-persistent` package (see +[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html)). -### 5. Fire up mitmproxy +### 4. Fire up mitmproxy. You probably want a command like this: @@ -118,24 +81,22 @@ mitmproxy --mode transparent --showhost The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 5. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the -default gateway. For a detailed walkthrough, have a look at the [tutorial for -transparently proxying VMs]({{< relref "howto-transparent-vms" >}}). +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). -## OpenBSD -### 1 [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) +## OpenBSD -### 2. Enable IP forwarding +### 1. Enable IP forwarding. {{< highlight bash >}} sudo sysctl -w net.inet.ip.forwarding=1 {{< / highlight >}} -### 3. Place the following two lines in **/etc/pf.conf** +### 2. Place the following two lines in **/etc/pf.conf**. {{< highlight none >}} mitm_if = "re2" @@ -146,19 +107,19 @@ These rules tell pf to divert all traffic from `$mitm_if` destined for port 80 or 443 to the local mitmproxy instance running on port 8080. You should replace `$mitm_if` value with the interface on which your test device will appear. -### 4. Enable the pf ruleset and enable it +### 3. Configure pf with the rules. {{< highlight bash >}} doas pfctl -f /etc/pf.conf {{< / highlight >}} -And now enable it: +### 4. And now enable it. {{< highlight bash >}} doas pfctl -e {{< / highlight >}} -### 5. Fire up mitmproxy +### 5. Fire up mitmproxy. You probably want a command like this: @@ -169,10 +130,11 @@ mitmproxy --mode transparent --showhost The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 6. Finally, configure your test device. + +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). -Set the test device up to use the host on which mitmproxy is running as the -default gateway. {{% note %}} @@ -195,15 +157,13 @@ packet filter from the OpenBSD project, which mitmproxy uses to implement transparent mode on OSX. Note that this means we don't support transparent mode for earlier versions of OSX. -### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}}) - -### 2. Enable IP forwarding +### 1. Enable IP forwarding. {{< highlight bash >}} sudo sysctl -w net.inet.ip.forwarding=1 {{< / highlight >}} -### 3. Place the following two lines in a file called, say, **pf.conf** +### 2. Place the following two lines in a file called, say, **pf.conf**. {{< highlight none >}} @@ -212,21 +172,21 @@ rdr on en0 inet proto tcp to any port {80, 443} -> 127.0.0.1 port 8080 These rules tell pf to redirect all traffic destined for port 80 or 443 to the local mitmproxy instance running on port 8080. You should replace -`en2` with the interface on which your test device will appear. +`en0` with the interface on which your test device will appear. -### 4. Configure pf with the rules +### 3. Configure pf with the rules. {{< highlight bash >}} sudo pfctl -f pf.conf {{< / highlight >}} -### 5. And now enable it +### 4. And now enable it. {{< highlight bash >}} sudo pfctl -e {{< / highlight >}} -### 6. Configure sudoers to allow mitmproxy to access pfctl +### 5. Configure sudoers to allow mitmproxy to access pfctl. Edit the file **/etc/sudoers** on your system as root. Add the following line to the end of the file: @@ -240,7 +200,7 @@ state` as root without a password. This only allows inspection of the state table, so should not be an undue security risk. If you're special feel free to tighten the restriction up to the user running mitmproxy. -### 7. Fire up mitmproxy +### 6. Fire up mitmproxy. You probably want a command like this: @@ -251,10 +211,10 @@ mitmproxy --mode transparent --showhost The `--mode transparent` flag turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. -### 6. Finally, configure your test device +### 7. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the -default gateway. +Set the test device up to use the host on which mitmproxy is running as the default gateway and +[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). {{% note %}} Note that the **rdr** rules in the pf.conf given above only apply to @@ -267,3 +227,38 @@ flexible to cater for a range of creative possibilities, like intercepting traffic emanating from VMs. See the **pf.conf** man page for more. {{% /note %}} + + +## "Full" transparent mode on Linux + +By default mitmproxy will use its own local IP address for its server-side +connections. In case this isn't desired, the --spoof-source-address argument can +be used to use the client's IP address for server-side connections. The +following config is required for this mode to work: + +{{< highlight bash >}} +CLIENT_NET=192.168.1.0/24 +TABLE_ID=100 +MARK=1 + +echo "$TABLE_ID mitmproxy" >> /etc/iproute2/rt_tables +iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK +iptables -t nat \ + -A PREROUTING -p tcp -s $CLIENT_NET \ + --match multiport --dports 80,443 -j \ + REDIRECT --to-port 8080 + +ip rule add fwmark $MARK lookup $TABLE_ID +ip route add local $CLIENT_NET dev lo table $TABLE_ID +{{< / highlight >}} + +This mode does require root privileges though. There's a wrapper in the examples +directory called 'mitmproxy_shim.c', which will enable you to use this mode with +dropped privileges. It can be used as follows: + +{{< highlight bash >}} +gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap +sudo chown root:root mitmproxy_shim +sudo chmod u+s mitmproxy_shim +./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address +{{< / highlight >}} diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md new file mode 100644 index 00000000..588223ac --- /dev/null +++ b/docs/src/content/howto-wireshark-tls.md @@ -0,0 +1,28 @@ +--- +title: "Wireshark and SSL/TLS" +menu: + howto: + weight: 1 +--- + +# Wireshark and SSL/TLS Master Secrets + +The SSL/SSL master keys can be logged by mitmproxy so that external programs can +decrypt SSL/TLS connections both from and to the proxy. Recent versions of +Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. + +Key logging is enabled by setting the environment variable `SSLKEYLOGFILE` so +that it points to a writable text file: +{{< highlight bash >}} +SSLKEYLOGFILE="$PWD/.mitmproxy/sslkeylogfile.txt" mitmproxy +{{< / highlight >}} +You can also `export` this environment variable to make it persistent for all applications started from your current shell session. + +You can specify the key file path in Wireshark via `Edit -> Preferences -> +Protocols -> SSL -> (Pre)-Master-Secret log filename`. If your SSLKEYLOGFILE +does not exist yet, just create an empty text file, so you can select it in +Wireshark (or run mitmproxy to create and collect master secrets). + +Note that `SSLKEYLOGFILE` is respected by other programs as well, e.g., Firefox +and Chrome. If this creates any issues, you can use `MITMPROXY_SSLKEYLOGFILE` +instead without affecting other applications. diff --git a/docs/src/content/overview-installation.md b/docs/src/content/overview-installation.md index 00941cb8..d69805db 100644 --- a/docs/src/content/overview-installation.md +++ b/docs/src/content/overview-installation.md @@ -73,14 +73,14 @@ security considerations apply as for our binary packages. ## Installation on Linux via pip3 -Please make sure to install Python 3.5 (or higher) and pip3 for your +Please make sure to install Python 3.6 (or higher) and pip3 for your distribution. If your distribution does not provide a suitable Python version, you can use [pyenv](https://github.com/yyuu/pyenv) to get a recent Python environment. {{< highlight bash >}} -sudo apt install python3-pip # Debian 8 or higher, Ubuntu 16.04 or higher -sudo dnf install python3-pip # Fedora 24 or higher +sudo apt install python3-pip # Debian 10 or higher, Ubuntu 17.10 or higher +sudo dnf install python3-pip # Fedora 26 or higher sudo pacman -S python-pip # Arch Linux {{< / highlight >}} @@ -98,7 +98,7 @@ sudo pip3 install mitmproxy ## Installation on Windows via pip3 -First, install the latest version of Python 3.5 or higher from the +First, install the latest version of Python 3.6 or higher from the [Python website](https://www.python.org/downloads/windows/). During installation, make sure to select Add Python to PATH. There are no other dependencies on Windows. diff --git a/docs/src/generated/.gitkeep b/docs/src/generated/.gitkeep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/docs/src/generated/.gitkeep diff --git a/docs/src/layouts/_default/single.html b/docs/src/layouts/_default/single.html index 4a8baf53..801b6341 100644 --- a/docs/src/layouts/_default/single.html +++ b/docs/src/layouts/_default/single.html @@ -1,10 +1,12 @@ -{{ partial "header.html" . }} -<div class="columns"> - <div class="column is-one-quarter sidebody"> - {{ partial "sidebar.html" . }} +{{ partial "header" . }} +<div class="columns container is-marginless"> + <div id="sidebar" class="column is-one-quarter"> + {{ partial "sidebar" . }} </div> - <div class="column content mainbody"> - {{.Content}} + <div id="main" class="column content"> + {{ partial "outdated" . }} + {{ partial "edit-on-github" . }} + {{ partial "add-anchors" .Content}} </div> </div> {{ partial "footer.html" . }} diff --git a/docs/src/layouts/index.html b/docs/src/layouts/index.html deleted file mode 100644 index 4a8baf53..00000000 --- a/docs/src/layouts/index.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ partial "header.html" . }} -<div class="columns"> - <div class="column is-one-quarter sidebody"> - {{ partial "sidebar.html" . }} - </div> - <div class="column content mainbody"> - {{.Content}} - </div> -</div> -{{ partial "footer.html" . }} diff --git a/docs/src/layouts/partials/add-anchors.html b/docs/src/layouts/partials/add-anchors.html new file mode 100644 index 00000000..f7050f7f --- /dev/null +++ b/docs/src/layouts/partials/add-anchors.html @@ -0,0 +1 @@ +{{ . | replaceRE "(<h[1-9] id=\"(.+?)\".*?>)(.+?</h[1-9]>)" "${1}<a class=\"anchor\" href=\"#${2}\"># </a>${3}" | safeHTML }} diff --git a/docs/src/layouts/partials/edit-on-github.html b/docs/src/layouts/partials/edit-on-github.html new file mode 100644 index 00000000..d2c3098c --- /dev/null +++ b/docs/src/layouts/partials/edit-on-github.html @@ -0,0 +1,9 @@ +{{ if and .IsPage (not (getenv "DOCS_ARCHIVE")) }} +<a class="button is-small is-outlined is-link is-pulled-right" + target="_blank" + href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}" +> + Edit on GitHub +</a> +{{ end }} + diff --git a/docs/src/layouts/partials/outdated.html b/docs/src/layouts/partials/outdated.html new file mode 100644 index 00000000..5b3dd6ed --- /dev/null +++ b/docs/src/layouts/partials/outdated.html @@ -0,0 +1,9 @@ +{{- if (getenv "DOCS_ARCHIVE") -}} +<article class="message is-warning"> + <div class="message-body"> + You are not viewing the most up to date version of the documentation. + Click <a href="https://docs.mitmproxy.org/stable{{ .Page.URL }}">here</a> + to view the latest version. + </div> +</article> +{{- end -}} diff --git a/docs/src/layouts/partials/sidebar.html b/docs/src/layouts/partials/sidebar.html index ef853fc6..5ea41c12 100644 --- a/docs/src/layouts/partials/sidebar.html +++ b/docs/src/layouts/partials/sidebar.html @@ -1,24 +1,22 @@ -<div class="sidebar"> - <div class="brand"> - <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs"> - </div> - <div class="version"> - <span class="tag is-info is-rounded is-medium">v3.x</span> - </div> - <aside class="menu"> - <p class="menu-label"> Overview </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "overview") }} +<div class="brand"> + <a href="https://mitmproxy.org/"> + <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs"/> + </a> - <p class="menu-label">Core concepts</p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "concepts") }} +</div> +<nav class="menu"> + <p class="menu-label"> Overview </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "overview") }} - <p class="menu-label"> Addon Development </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "addons") }} + <p class="menu-label">Core concepts</p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "concepts") }} - <p class="menu-label"> HOWTOs </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "howto") }} + <p class="menu-label"> Addon Development </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "addons") }} - <p class="menu-label"> Tutorials </p> - {{ partial "sidemenu" (dict "ctx" . "menuname" "tutes") }} - </aside> -</div>
\ No newline at end of file + <p class="menu-label"> HOWTOs </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "howto") }} + + <p class="menu-label"> Tutorials </p> + {{ partial "sidemenu" (dict "ctx" . "menuname" "tutes") }} +</nav> diff --git a/docs/src/layouts/shortcodes/readfile.html b/docs/src/layouts/shortcodes/readfile.html new file mode 100644 index 00000000..6860b0f1 --- /dev/null +++ b/docs/src/layouts/shortcodes/readfile.html @@ -0,0 +1,6 @@ +{{$file := .Get "file"}} +{{- if eq (.Get "markdown") "true" -}} +{{- $file | readFile | markdownify -}} +{{- else -}} +{{ $file | readFile | safeHTML }} +{{- end -}} diff --git a/docs/src/static/logo-docs.png b/docs/src/static/logo-docs.png Binary files differindex b37dbd85..a46016dd 100644 --- a/docs/src/static/logo-docs.png +++ b/docs/src/static/logo-docs.png diff --git a/docs/src/themes/mitmproxydocs/layouts/partials/footer.html b/docs/src/themes/mitmproxydocs/layouts/partials/footer.html index 308b1d01..dc9ddc85 100644 --- a/docs/src/themes/mitmproxydocs/layouts/partials/footer.html +++ b/docs/src/themes/mitmproxydocs/layouts/partials/footer.html @@ -1,2 +1,3 @@ +{{ template "_internal/google_analytics_async.html" . }} </body> </html> diff --git a/docs/src/themes/mitmproxydocs/static/css/style.css b/docs/src/themes/mitmproxydocs/static/css/style.css index 14823447..d42e307e 100644 --- a/docs/src/themes/mitmproxydocs/static/css/style.css +++ b/docs/src/themes/mitmproxydocs/static/css/style.css @@ -250,6 +250,21 @@ .chroma .gu { color: #75715e; } +.badge { + color: #fff; + background-color: #6c757d; + display: inline-block; + padding: .25em .4em; + font-size: 75%; + font-weight: 1; + line-height: 1; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: 0.25rem; } + .badge:empty { + display: none; } + @keyframes spinAround { from { transform: rotate(0deg); } @@ -6717,10 +6732,17 @@ label.panel-block { background-color: whitesmoke; padding: 3rem 1.5rem 6rem; } -.sidebody { - height: 100vh; - overflow-x: hidden; - overflow-y: scroll; } +#sidebar { + background-color: #eee; + border-right: 1px solid #c1c1c1; + box-shadow: 0 0 20px rgba(50, 50, 50, 0.2) inset; + padding: 1.75rem; } + #sidebar .brand { + padding: 1rem 0; + text-align: center; } + +#main { + padding: 3rem; } .example { margin-bottom: 1em; } @@ -6731,21 +6753,6 @@ label.panel-block { width: 100%; text-align: right; } -.sidebar .version { - padding: 1em; } - -.sidebar .brand { - background-color: #303030; - color: #c0c0c0; - padding: 1em; - top: 0; } - -.sidebar .menu { - padding: 1em; } - -.mainbody { - padding: 3em; } - code { color: #1a9f1a; font-size: 0.875em; @@ -6754,3 +6761,26 @@ code { .content h2 { padding-top: 1em; border-top: 1px solid #c0c0c0; } + +h1 .anchor, h2 .anchor, h3 .anchor, h4 .anchor, h5 .anchor, h6 .anchor { + display: inline-block; + width: 0; + margin-left: -1.5rem; + margin-right: 1.5rem; + transition: all 100ms ease-in-out; + opacity: 0; } + +h1:hover .anchor, h2:hover .anchor, h3:hover .anchor, h4:hover .anchor, h5:hover .anchor, h6:hover .anchor { + opacity: 1; } + +h1:target, h2:target, h3:target, h4:target, h5:target, h6:target { + color: #C93312; } + h1:target .anchor, h2:target .anchor, h3:target .anchor, h4:target .anchor, h5:target .anchor, h6:target .anchor { + opacity: 1; + color: #C93312; } + +.footnotes p { + display: inline; } + +figure.has-border img { + box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25); } diff --git a/docs/style/badge.scss b/docs/style/badge.scss new file mode 100644 index 00000000..8082f6c7 --- /dev/null +++ b/docs/style/badge.scss @@ -0,0 +1,18 @@ +.badge { + color: #fff; + background-color: #6c757d; + display: inline-block; + padding: .25em .4em; + font-size: 75%; + font-weight: 1; + line-height: 1; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: 0.25rem; + + // Empty badges collapse automatically + &:empty { + display: none; + } +} diff --git a/docs/style/style.scss b/docs/style/style.scss index 2c66a4c9..b404dc42 100644 --- a/docs/style/style.scss +++ b/docs/style/style.scss @@ -1,4 +1,5 @@ @import "./syntax"; +@import "./badge"; $primary: #C93312; $warning-invert: #FFFFFF; @@ -10,10 +11,20 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox @import "../node_modules/bulma/sass/components/_all"; @import "../node_modules/bulma/sass/layout/_all"; -.sidebody { - height: 100vh; - overflow-x: hidden; - overflow-y: scroll; +#sidebar { + background-color: #eee; + border-right: 1px solid #c1c1c1; + box-shadow: 0 0 20px rgba(50, 50, 50, .2) inset; + padding: $column-gap + 1rem; + + .brand { + padding: 1rem 0; + text-align: center; + } +} + +#main { + padding: 3rem; } .example { @@ -28,30 +39,10 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox margin-bottom: 1em; } - -.sidebar { - .version { - padding: 1em; - } - .brand { - background-color: #303030; - color: #c0c0c0; - padding: 1em; - top: 0; - } - .menu { - padding: 1em; - } -} - -.mainbody { - padding: 3em; -} - code { - color: #1a9f1a; - font-size: 0.875em; - font-weight: normal; + color: #1a9f1a; + font-size: 0.875em; + font-weight: normal; } .content { @@ -60,3 +51,32 @@ code { border-top: 1px solid #c0c0c0; } } + +h1, h2, h3, h4, h5, h6 { + .anchor { + display: inline-block; + width: 0; + margin-left: -1.5rem; + margin-right: 1.5rem; + transition: all 100ms ease-in-out; + opacity: 0; + } + &:hover .anchor { + opacity: 1; + } + &:target { + color: $primary; + .anchor { + opacity: 1; + color: $primary + } + } +} + +.footnotes p { + display: inline; +} + +figure.has-border img { + box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25); +} diff --git a/docs/upload-archive b/docs/upload-archive new file mode 100755 index 00000000..3aaeb9be --- /dev/null +++ b/docs/upload-archive @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [[ $# -eq 0 ]] ; then + echo "Please supply a version, e.g. 'v3'" + exit 1 +fi + +# This script uploads docs to a specified archive version. + +SPATH="/archive/$1" + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "$SPATH/*" diff --git a/docs/upload-stable b/docs/upload-stable new file mode 100755 index 00000000..5aea7479 --- /dev/null +++ b/docs/upload-stable @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/stable/*" diff --git a/examples/addons/events.py b/examples/addons/events.py index d3c90430..674ab4c8 100644 --- a/examples/addons/events.py +++ b/examples/addons/events.py @@ -80,12 +80,12 @@ class Events: attribute. """ - def websocket_start(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_start(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has commenced. """ - def websocket_message(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_message(self, flow: mitmproxy.websocket.WebSocketFlow): """ Called when a WebSocket message is received from the client or server. The most recent message will be flow.messages[-1]. The @@ -93,12 +93,12 @@ class Events: messages, corresponding to the BINARY and TEXT frame types. """ - def websocket_error(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_error(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has had an error. """ - def websocket_end(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_end(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has ended. """ diff --git a/mitmproxy/addons/browser.py b/mitmproxy/addons/browser.py index 247c356b..2f8eeccd 100644 --- a/mitmproxy/addons/browser.py +++ b/mitmproxy/addons/browser.py @@ -15,6 +15,7 @@ def get_chrome_executable() -> typing.Optional[str]: r"C:\Program Files (x86)\Google\Application\chrome.exe", # Linux binary names from Python's webbrowser module. "google-chrome", + "google-chrome-stable", "chrome", "chromium", "chromium-browser", diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index f9874038..f7fbc0c8 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -129,7 +129,7 @@ class Cut: if isinstance(v, bytes): fp.write(strutils.always_str(v)) else: - fp.write("utf8") + fp.write(v) ctx.log.alert("Clipped single cut.") else: writer = csv.writer(fp) diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 45141576..114e882d 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -1,5 +1,5 @@ """ - This module manges and invokes typed commands. + This module manages and invokes typed commands. """ import inspect import types @@ -131,8 +131,13 @@ class CommandManager(mitmproxy.types._CommandBase): for i in dir(addon): if not i.startswith("__"): o = getattr(addon, i) - if hasattr(o, "command_path"): - self.add(o.command_path, o) + try: + is_command = hasattr(o, "command_path") + except Exception: + pass # hasattr may raise if o implements __getattr__. + else: + if is_command: + self.add(o.command_path, o) def add(self, path: str, func: typing.Callable): self.commands[path] = Command(self, path, func) diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 29ab6ab5..9c26b44f 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -1,18 +1,18 @@ -import time - import os +import time import typing import uuid -from mitmproxy import stateobject, exceptions from mitmproxy import certs +from mitmproxy import exceptions +from mitmproxy import stateobject from mitmproxy.net import tcp from mitmproxy.net import tls +from mitmproxy.utils import human from mitmproxy.utils import strutils class ClientConnection(tcp.BaseHandler, stateobject.StateObject): - """ A client connection @@ -72,11 +72,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): else: alpn = "" - return "<ClientConnection: {tls}{alpn}{host}:{port}>".format( + return "<ClientConnection: {tls}{alpn}{address}>".format( tls=tls, alpn=alpn, - host=self.address[0], - port=self.address[1], + address=human.format_address(self.address), ) def __eq__(self, other): @@ -161,7 +160,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): class ServerConnection(tcp.TCPClient, stateobject.StateObject): - """ A server connection @@ -209,11 +207,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ) else: alpn = "" - return "<ServerConnection: {tls}{alpn}{host}:{port}>".format( + return "<ServerConnection: {tls}{alpn}{address}>".format( tls=tls, alpn=alpn, - host=self.address[0], - port=self.address[1], + address=human.format_address(self.address), ) def __eq__(self, other): diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index beb210ca..79b049c9 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -1,4 +1,5 @@ import queue +import asyncio from mitmproxy import exceptions @@ -7,9 +8,10 @@ class Channel: The only way for the proxy server to communicate with the master is to use the channel it has been given. """ - def __init__(self, q, should_exit): - self.q = q + def __init__(self, loop, q, should_exit): + self.loop = loop self.should_exit = should_exit + self._q = q def ask(self, mtype, m): """ @@ -20,18 +22,11 @@ class Channel: exceptions.Kill: All connections should be closed immediately. """ m.reply = Reply(m) - self.q.put((mtype, m)) - while not self.should_exit.is_set(): - try: - # The timeout is here so we can handle a should_exit event. - g = m.reply.q.get(timeout=0.5) - except queue.Empty: # pragma: no cover - continue - if g == exceptions.Kill: - raise exceptions.Kill() - return g - m.reply._state = "committed" # suppress error message in __del__ - raise exceptions.Kill() + asyncio.run_coroutine_threadsafe(self._q.put((mtype, m)), self.loop) + g = m.reply.q.get() + if g == exceptions.Kill: + raise exceptions.Kill() + return g def tell(self, mtype, m): """ @@ -39,7 +34,7 @@ class Channel: then return immediately. """ m.reply = DummyReply() - self.q.put((mtype, m)) + asyncio.run_coroutine_threadsafe(self._q.put((mtype, m)), self.loop) NO_REPLY = object() # special object we can distinguish from a valid "None" reply. @@ -52,7 +47,8 @@ class Reply: """ def __init__(self, obj): self.obj = obj - self.q = queue.Queue() # type: queue.Queue + # Spawn an event loop in the current thread + self.q = queue.Queue() self._state = "start" # "start" -> "taken" -> "committed" diff --git a/mitmproxy/master.py b/mitmproxy/master.py index a5e948f6..372bb289 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -1,6 +1,7 @@ import threading import contextlib -import queue +import asyncio +import logging from mitmproxy import addonmanager from mitmproxy import options @@ -18,6 +19,13 @@ from mitmproxy.coretypes import basethread from . import ctx as mitmproxy_ctx +# Conclusively preventing cross-thread races on proxy shutdown turns out to be +# very hard. We could build a thread sync infrastructure for this, or we could +# wait until we ditch threads and move all the protocols into the async loop. +# Until then, silence non-critical errors. +logging.getLogger('asyncio').setLevel(logging.CRITICAL) + + class ServerThread(basethread.BaseThread): def __init__(self, server): self.server = server @@ -35,11 +43,19 @@ class Master: The master handles mitmproxy's main event loop. """ def __init__(self, opts): + self.event_queue = asyncio.Queue() + self.should_exit = threading.Event() + self.channel = controller.Channel( + asyncio.get_event_loop(), + self.event_queue, + self.should_exit, + ) + asyncio.ensure_future(self.main()) + asyncio.ensure_future(self.tick()) + self.options = opts or options.Options() # type: options.Options self.commands = command.CommandManager(self) self.addons = addonmanager.AddonManager(self) - self.event_queue = queue.Queue() - self.should_exit = threading.Event() self._server = None self.first_tick = True self.waiting_flows = [] @@ -50,9 +66,7 @@ class Master: @server.setter def server(self, server): - server.set_channel( - controller.Channel(self.event_queue, self.should_exit) - ) + server.set_channel(self.channel) self._server = server @contextlib.contextmanager @@ -71,7 +85,8 @@ class Master: mitmproxy_ctx.log = None mitmproxy_ctx.options = None - def tell(self, mtype, m): + # This is a vestigial function that will go away in a refactor very soon + def tell(self, mtype, m): # pragma: no cover m.reply = controller.DummyReply() self.event_queue.put((mtype, m)) @@ -86,38 +101,43 @@ class Master: if self.server: ServerThread(self.server).start() - def run(self): - self.start() - try: - while not self.should_exit.is_set(): - self.tick(0.1) - finally: - self.shutdown() + async def main(self): + while True: + try: + mtype, obj = await self.event_queue.get() + except RuntimeError: + return + if mtype not in eventsequence.Events: # pragma: no cover + raise exceptions.ControlException("Unknown event %s" % repr(mtype)) + self.addons.handle_lifecycle(mtype, obj) + self.event_queue.task_done() - def tick(self, timeout): + async def tick(self): if self.first_tick: self.first_tick = False self.addons.trigger("running") - self.addons.trigger("tick") - changed = False + while True: + if self.should_exit.is_set(): + asyncio.get_event_loop().stop() + return + self.addons.trigger("tick") + await asyncio.sleep(0.1) + + def run(self): + self.start() + asyncio.ensure_future(self.tick()) + loop = asyncio.get_event_loop() try: - mtype, obj = self.event_queue.get(timeout=timeout) - if mtype not in eventsequence.Events: - raise exceptions.ControlException( - "Unknown event %s" % repr(mtype) - ) - self.addons.handle_lifecycle(mtype, obj) - self.event_queue.task_done() - changed = True - except queue.Empty: - pass - return changed + loop.run_forever() + finally: + self.shutdown() + loop.close() + self.addons.trigger("done") def shutdown(self): if self.server: self.server.shutdown() self.should_exit.set() - self.addons.trigger("done") def _change_reverse_host(self, f): """ @@ -199,12 +219,7 @@ class Master: host = f.request.headers.pop(":authority") f.request.headers.insert(0, "host", host) - rt = http_replay.RequestReplayThread( - self.options, - f, - self.event_queue, - self.should_exit - ) + rt = http_replay.RequestReplayThread(self.options, f, self.channel) rt.start() # pragma: no cover if block: rt.join() diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 6b4041f6..959fdd33 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -429,10 +429,7 @@ class Request(message.Message): def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: - try: - return tuple(mitmproxy.net.http.url.decode(self.content.decode())) - except ValueError: - pass + return tuple(mitmproxy.net.http.url.decode(self.get_text(strict=False))) return () def _set_urlencoded_form(self, form_data): @@ -441,7 +438,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() + self.content = mitmproxy.net.http.url.encode(form_data, self.get_text(strict=False)).encode() @property def urlencoded_form(self): diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index 85217794..b842f11f 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -19,7 +19,7 @@ from mitmproxy.coretypes import basethread socket_fileobject = socket.SocketIO # workaround for https://bugs.python.org/issue29515 -# Python 3.5 and 3.6 for Windows is missing a constant +# Python 3.6 for Windows is missing a constant IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41) EINTR = 4 @@ -547,7 +547,6 @@ class Counter: class TCPServer: - request_queue_size = 20 def __init__(self, address): self.address = address @@ -580,7 +579,7 @@ class TCPServer: self.socket.bind(self.address) self.address = self.socket.getsockname() - self.socket.listen(self.request_queue_size) + self.socket.listen() self.handler_counter = Counter() def connection_thread(self, connection, client_address): diff --git a/mitmproxy/platform/osx.py b/mitmproxy/platform/osx.py index f9de1fbf..9b150679 100644 --- a/mitmproxy/platform/osx.py +++ b/mitmproxy/platform/osx.py @@ -32,5 +32,5 @@ def original_addr(csock): if insufficient_priv: raise RuntimeError( "Insufficient privileges to access pfctl. " - "See http://docs.mitmproxy.org/en/latest/transparent/osx.html for details.") + "See https://mitmproxy.org/docs/latest/howto-transparent/#macos for details.") return pf.lookup(peer[0], peer[1], stxt) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 439beb3d..410ab701 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -53,7 +53,7 @@ class ProxyConfig: if not os.path.exists(os.path.dirname(certstore_path)): raise exceptions.OptionsError( "Certificate Authority parent directory does not exist: %s" % - os.path.dirname(options.cadir) + os.path.dirname(certstore_path) ) self.certstore = certs.CertStore.from_store( certstore_path, diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 99286fa5..24b411e8 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -333,8 +333,20 @@ class HttpLayer(base.Layer): f.request.scheme ) - try: + def get_response(): self.send_request_headers(f.request) + if f.request.stream: + chunks = self.read_request_body(f.request) + if callable(f.request.stream): + chunks = f.request.stream(chunks) + self.send_request_body(f.request, chunks) + else: + self.send_request_body(f.request, [f.request.data.content]) + + f.response = self.read_response_headers() + + try: + get_response() except exceptions.NetlibException as e: self.log( "server communication error: %s" % repr(e), @@ -357,22 +369,17 @@ class HttpLayer(base.Layer): raise exceptions.ProtocolException( "First and only attempt to get response via HTTP2 failed." ) + elif f.request.stream: + # We may have already consumed some request chunks already, + # so all we can do is signal downstream that upstream closed the connection. + self.send_error_response(408, "Request Timeout") + f.error = flow.Error(repr(e)) + self.channel.ask("error", f) + return False self.disconnect() self.connect() - self.send_request_headers(f.request) - - # This is taken out of the try except block because when streaming - # we can't send the request body while retrying as the generator gets exhausted - if f.request.stream: - chunks = self.read_request_body(f.request) - if callable(f.request.stream): - chunks = f.request.stream(chunks) - self.send_request_body(f.request, chunks) - else: - self.send_request_body(f.request, [f.request.data.content]) - - f.response = self.read_response_headers() + get_response() # call the appropriate script hook - this is an opportunity for # an inline script to set f.stream = True diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 0f3be1ea..b2cca2b1 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -1,7 +1,3 @@ -import queue -import threading -import typing - from mitmproxy import log from mitmproxy import controller from mitmproxy import exceptions @@ -25,20 +21,12 @@ class RequestReplayThread(basethread.BaseThread): self, opts: options.Options, f: http.HTTPFlow, - event_queue: typing.Optional[queue.Queue], - should_exit: threading.Event + channel: controller.Channel, ) -> None: - """ - event_queue can be a queue or None, if no scripthooks should be - processed. - """ self.options = opts self.f = f f.live = True - if event_queue: - self.channel = controller.Channel(event_queue, should_exit) - else: - self.channel = None + self.channel = channel super().__init__( "RequestReplay (%s)" % f.request.url ) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 9420c105..650a9366 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -30,12 +30,14 @@ class FlowViewHeader(urwid.WidgetWrap): self.focus_changed() def focus_changed(self): + cols, _ = self.master.ui.get_cols_rows() if self.master.view.focus.flow: self._w = common.format_flow( self.master.view.focus.flow, False, extended=True, - hostheader=self.master.options.showhost + hostheader=self.master.options.showhost, + max_url_len=cols, ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 5da6ef0b..de660b17 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -1,3 +1,4 @@ +import asyncio import mailcap import mimetypes import os @@ -94,7 +95,9 @@ class ConsoleMaster(master.Master): self.start_err = entry else: signals.status_message.send( - message=(entry.level, "{}: {}".format(entry.level.title(), entry.msg)), + message=(entry.level, + "{}: {}".format(entry.level.title(), + str(entry.msg).lstrip())), expire=5 ) @@ -180,12 +183,6 @@ class ConsoleMaster(master.Master): ) self.ui.clear() - def ticker(self, *userdata): - changed = self.tick(timeout=0) - if changed: - self.loop.draw_screen() - self.loop.set_alarm_in(0.01, self.ticker) - def inject_key(self, key): self.loop.process_input([key]) @@ -204,6 +201,7 @@ class ConsoleMaster(master.Master): ) self.loop = urwid.MainLoop( urwid.SolidFill("x"), + event_loop=urwid.AsyncioEventLoop(loop=asyncio.get_event_loop()), screen = self.ui, handle_mouse = self.options.console_mouse, ) @@ -212,8 +210,6 @@ class ConsoleMaster(master.Master): self.loop.widget = self.window self.window.refresh() - self.loop.set_alarm_in(0.01, self.ticker) - if self.start_err: def display_err(*_): self.sig_add_log(None, self.start_err) @@ -234,6 +230,7 @@ class ConsoleMaster(master.Master): finally: sys.stderr.flush() super().shutdown() + self.addons.trigger("done") def shutdown(self): raise urwid.ExitMainLoop diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 8553a66f..fa987e94 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -49,7 +49,8 @@ class ActionBar(urwid.WidgetWrap): def sig_message(self, sender, message, expire=1): if self.prompting: return - w = urwid.Text(message) + cols, _ = self.master.ui.get_cols_rows() + w = urwid.Text(self.shorten_message(message, cols)) self._w = w if expire: def cb(*args): @@ -60,6 +61,36 @@ class ActionBar(urwid.WidgetWrap): def prep_prompt(self, p): return p.strip() + ": " + def shorten_message(self, msg, max_width): + """ + Shorten message so that it fits into a single line in the statusbar. + """ + if isinstance(msg, tuple): + disp_attr, msg_text = msg + elif isinstance(msg, str): + disp_attr, msg_text = None, msg + else: + return msg + msg_end = "\u2026" # unicode ellipsis for the end of shortened message + prompt = "(more in eventlog)" + + msg_lines = msg_text.split("\n") + first_line = msg_lines[0] + if len(msg_lines) > 1: + # First line of messages with a few lines must end with prompt. + line_length = len(first_line) + len(prompt) + else: + line_length = len(first_line) + + if line_length > max_width: + shortening_index = max(0, max_width - len(prompt) - len(msg_end)) + first_line = first_line[:shortening_index] + msg_end + else: + if len(msg_lines) == 1: + prompt = "" + + return [(disp_attr, first_line), ("warn", prompt)] + def sig_prompt(self, sender, prompt, text, callback, args=()): signals.focus.send(self, section="footer") self._w = urwid.Edit(self.prep_prompt(prompt), text or "") @@ -160,9 +191,7 @@ class StatusBar(urwid.WidgetWrap): r.append(("heading_key", "H")) r.append("eaders]") if len(self.master.options.replacements): - r.append("[") - r.append(("heading_key", "R")) - r.append("eplacing]") + r.append("[%d replacements]" % len(self.master.options.replacements)) if creplay.count(): r.append("[") r.append(("heading_key", "cplayback")) @@ -197,10 +226,8 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "u")) r.append(":%s]" % self.master.options.stickyauth) - if self.master.options.console_default_contentview != "auto": - r.append("[") - r.append(("heading_key", "M")) - r.append(":%s]" % self.master.options.console_default_contentview) + if self.master.options.console_default_contentview != 'auto': + r.append("[contentview:%s]" % (self.master.options.console_default_contentview)) if self.master.options.has_changed("view_order"): r.append("[") r.append(("heading_key", "o")) diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index f2b6d3f4..7669299c 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -294,7 +294,7 @@ class Window(urwid.Frame): if not k: if args[1] == "mouse drag": signals.status_message.send( - message = "Hold down fn, shift, alt or ctrl to select text or use the --no-mouse parameter.", + message = "Hold down fn, shift, alt or ctrl to select text or use the --set console_mouse=false parameter.", expire = 1 ) elif args[1] == "mouse press" and args[2] == 4: diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 330060f7..53c236bb 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -1,12 +1,13 @@ from __future__ import print_function # this is here for the version check to work on Python 2. +import asyncio import sys -if sys.version_info < (3, 5): +if sys.version_info < (3, 6): # This must be before any mitmproxy imports, as they already break! # Keep all other imports below with the 'noqa' magic comment. print("#" * 49, file=sys.stderr) - print("# mitmproxy only supports Python 3.5 and above! #", file=sys.stderr) + print("# mitmproxy requires Python 3.6 or higher! #", file=sys.stderr) print("#" * 49, file=sys.stderr) import argparse # noqa @@ -117,8 +118,10 @@ def run( def cleankill(*args, **kwargs): master.shutdown() - signal.signal(signal.SIGTERM, cleankill) + loop = asyncio.get_event_loop() + for signame in ('SIGINT', 'SIGTERM'): + loop.add_signal_handler(getattr(signal, signame), master.shutdown) master.run() except exceptions.OptionsError as e: print("%s: %s" % (sys.argv[0], e), file=sys.stderr) diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 4c597f0e..b7eddcce 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -2,6 +2,8 @@ import webbrowser import tornado.httpserver import tornado.ioloop +from tornado.platform.asyncio import AsyncIOMainLoop + from mitmproxy import addons from mitmproxy import log from mitmproxy import master @@ -102,6 +104,7 @@ class WebMaster(master.Master): ) def run(self): # pragma: no cover + AsyncIOMainLoop().install() iol = tornado.ioloop.IOLoop.instance() @@ -109,7 +112,6 @@ class WebMaster(master.Master): http_server.listen(self.options.web_port, self.options.web_iface) iol.add_callback(self.start) - tornado.ioloop.PeriodicCallback(lambda: self.tick(timeout=0), 5).start() web_url = "http://{}:{}/".format(self.options.web_iface, self.options.web_port) self.add_log( diff --git a/mitmproxy/tools/web/static/app.js b/mitmproxy/tools/web/static/app.js index ace3106d..ac3be38c 100644 --- a/mitmproxy/tools/web/static/app.js +++ b/mitmproxy/tools/web/static/app.js @@ -86,7 +86,7 @@ "use strict";function _interopRequireWildcard(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&(t[r]=e[r]);return t.default=e,t}function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function FileMenu(e){var t=e.clearFlows,r=e.loadFlows,a=e.saveFlows,l=e.openModal;return _react2.default.createElement(_Dropdown2.default,{className:"pull-left",btnClass:"special",text:"mitmproxy"},_react2.default.createElement("a",{href:"#",onClick:function(e){return FileMenu.onNewClick(e,t)}},_react2.default.createElement("i",{className:"fa fa-fw fa-file"}),"Â New"),_react2.default.createElement(_FileChooser2.default,{icon:"fa-folder-open",text:"Â Open...",onOpenFile:function(e){return r(e)}}),_react2.default.createElement("a",{href:"#",onClick:function(e){e.preventDefault(),a()}},_react2.default.createElement("i",{className:"fa fa-fw fa-floppy-o"}),"Â Save..."),_react2.default.createElement(_HideInStatic2.default,null,_react2.default.createElement("a",{href:"#",onClick:function(e){e.preventDefault(),l()}},_react2.default.createElement("i",{className:"fa fa-fw fa-cog"}),"Â Options"),_react2.default.createElement(_Dropdown.Divider,null),_react2.default.createElement("a",{href:"http://mitm.it/",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-fw fa-external-link"}),"Â Install Certificates...")))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.FileMenu=FileMenu;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_reactRedux=require("react-redux"),_FileChooser=require("../common/FileChooser"),_FileChooser2=_interopRequireDefault(_FileChooser),_Dropdown=require("../common/Dropdown"),_Dropdown2=_interopRequireDefault(_Dropdown),_flows=require("../../ducks/flows"),flowsActions=_interopRequireWildcard(_flows),_modal=require("../../ducks/ui/modal"),modalActions=_interopRequireWildcard(_modal),_HideInStatic=require("../common/HideInStatic"),_HideInStatic2=_interopRequireDefault(_HideInStatic);FileMenu.propTypes={clearFlows:_propTypes2.default.func.isRequired,loadFlows:_propTypes2.default.func.isRequired,saveFlows:_propTypes2.default.func.isRequired,openModal:_propTypes2.default.func.isRequired},FileMenu.onNewClick=function(e,t){e.preventDefault(),confirm("Delete all flows?")&&t()},exports.default=(0,_reactRedux.connect)(null,{clearFlows:flowsActions.clear,loadFlows:flowsActions.upload,saveFlows:flowsActions.download,openModal:function(){return modalActions.setActiveModal("OptionModal")}})(FileMenu); },{"../../ducks/flows":57,"../../ducks/ui/modal":65,"../common/Dropdown":48,"../common/FileChooser":49,"../common/HideInStatic":50,"prop-types":"prop-types","react":"react","react-redux":"react-redux"}],30:[function(require,module,exports){ -"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_utils=require("../../utils"),FilterDocs=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={doc:t.doc},n}return _inherits(t,e),_createClass(t,[{key:"componentWillMount",value:function(){var e=this;t.xhr||(t.xhr=(0,_utils.fetchApi)("/filter-help").then(function(e){return e.json()}),t.xhr.catch(function(){t.xhr=null})),this.state.doc||t.xhr.then(function(r){t.doc=r,e.setState({doc:r})})}},{key:"render",value:function(){var e=this,t=this.state.doc;return t?_react2.default.createElement("table",{className:"table table-condensed"},_react2.default.createElement("tbody",null,t.commands.map(function(t){return _react2.default.createElement("tr",{key:t[1],onClick:function(r){return e.props.selectHandler(t[0].split(" ")[0]+" ")}},_react2.default.createElement("td",null,t[0].replace(" ","Â ")),_react2.default.createElement("td",null,t[1]))}),_react2.default.createElement("tr",{key:"docs-link"},_react2.default.createElement("td",{colSpan:"2"},_react2.default.createElement("a",{href:"http://docs.mitmproxy.org/en/stable/features/filters.html",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-external-link"}),"Â mitmproxy docs"))))):_react2.default.createElement("i",{className:"fa fa-spinner fa-spin"})}}]),t}(_react.Component);FilterDocs.xhr=null,FilterDocs.doc=null,exports.default=FilterDocs; +"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_utils=require("../../utils"),FilterDocs=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={doc:t.doc},n}return _inherits(t,e),_createClass(t,[{key:"componentWillMount",value:function(){var e=this;t.xhr||(t.xhr=(0,_utils.fetchApi)("/filter-help").then(function(e){return e.json()}),t.xhr.catch(function(){t.xhr=null})),this.state.doc||t.xhr.then(function(r){t.doc=r,e.setState({doc:r})})}},{key:"render",value:function(){var e=this,t=this.state.doc;return t?_react2.default.createElement("table",{className:"table table-condensed"},_react2.default.createElement("tbody",null,t.commands.map(function(t){return _react2.default.createElement("tr",{key:t[1],onClick:function(r){return e.props.selectHandler(t[0].split(" ")[0]+" ")}},_react2.default.createElement("td",null,t[0].replace(" ","Â ")),_react2.default.createElement("td",null,t[1]))}),_react2.default.createElement("tr",{key:"docs-link"},_react2.default.createElement("td",{colSpan:"2"},_react2.default.createElement("a",{href:"https://mitmproxy.org/docs/latest/concepts-filters/",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-external-link"}),"Â mitmproxy docs"))))):_react2.default.createElement("i",{className:"fa fa-spinner fa-spin"})}}]),t}(_react.Component);FilterDocs.xhr=null,FilterDocs.doc=null,exports.default=FilterDocs; },{"../../utils":71,"react":"react"}],31:[function(require,module,exports){ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var o=t[r];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(e,o.key,o)}}return function(t,r,o){return r&&e(t.prototype,r),o&&e(t,o),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_reactDom=require("react-dom"),_reactDom2=_interopRequireDefault(_reactDom),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames),_utils=require("../../utils.js"),_filt=require("../../filt/filt"),_filt2=_interopRequireDefault(_filt),_FilterDocs=require("./FilterDocs"),_FilterDocs2=_interopRequireDefault(_FilterDocs),FilterInput=function(e){function t(e,r){_classCallCheck(this,t);var o=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return o.state={value:o.props.value,focus:!1,mousefocus:!1},o.onChange=o.onChange.bind(o),o.onFocus=o.onFocus.bind(o),o.onBlur=o.onBlur.bind(o),o.onKeyDown=o.onKeyDown.bind(o),o.onMouseEnter=o.onMouseEnter.bind(o),o.onMouseLeave=o.onMouseLeave.bind(o),o.selectFilter=o.selectFilter.bind(o),o}return _inherits(t,e),_createClass(t,[{key:"componentWillReceiveProps",value:function(e){this.setState({value:e.value})}},{key:"isValid",value:function(e){try{var t=null==e?this.state.value:e;return t&&_filt2.default.parse(t),!0}catch(e){return!1}}},{key:"getDesc",value:function(){if(!this.state.value)return _react2.default.createElement(_FilterDocs2.default,{selectHandler:this.selectFilter});try{return _filt2.default.parse(this.state.value).desc}catch(e){return""+e}}},{key:"onChange",value:function(e){var t=e.target.value;this.setState({value:t}),this.isValid(t)&&this.props.onChange(t)}},{key:"onFocus",value:function(){this.setState({focus:!0})}},{key:"onBlur",value:function(){this.setState({focus:!1})}},{key:"onMouseEnter",value:function(){this.setState({mousefocus:!0})}},{key:"onMouseLeave",value:function(){this.setState({mousefocus:!1})}},{key:"onKeyDown",value:function(e){e.keyCode!==_utils.Key.ESC&&e.keyCode!==_utils.Key.ENTER||(this.blur(),this.setState({mousefocus:!1})),e.stopPropagation()}},{key:"selectFilter",value:function(e){this.setState({value:e}),_reactDom2.default.findDOMNode(this.refs.input).focus()}},{key:"blur",value:function(){_reactDom2.default.findDOMNode(this.refs.input).blur()}},{key:"select",value:function(){_reactDom2.default.findDOMNode(this.refs.input).select()}},{key:"render",value:function(){var e=this.props,t=e.type,r=e.color,o=e.placeholder,n=this.state,a=n.value,s=n.focus,u=n.mousefocus;return _react2.default.createElement("div",{className:(0,_classnames2.default)("filter-input input-group",{"has-error":!this.isValid()})},_react2.default.createElement("span",{className:"input-group-addon"},_react2.default.createElement("i",{className:"fa fa-fw fa-"+t,style:{color:r}})),_react2.default.createElement("input",{type:"text",ref:"input",placeholder:o,className:"form-control",value:a,onChange:this.onChange,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown}),(s||u)&&_react2.default.createElement("div",{className:"popover bottom",onMouseEnter:this.onMouseEnter,onMouseLeave:this.onMouseLeave},_react2.default.createElement("div",{className:"arrow"}),_react2.default.createElement("div",{className:"popover-content"},this.getDesc())))}}]),t}(_react.Component);exports.default=FilterInput; @@ -137,7 +137,7 @@ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function Button(e){var t=e.onClick,r=e.children,a=e.icon,s=e.disabled,l=e.className,i=e.title;return _react2.default.createElement("div",{className:(0,_classnames2.default)(l,"btn btn-default"),onClick:!s&&t,disabled:s,title:i},a&&_react2.default.createElement("i",{className:"fa fa-fw "+a}),r)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=Button;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames);Button.propTypes={onClick:_propTypes2.default.func.isRequired,children:_propTypes2.default.node.isRequired,icon:_propTypes2.default.string,title:_propTypes2.default.string}; },{"classnames":"classnames","prop-types":"prop-types","react":"react"}],47:[function(require,module,exports){ -"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function DocsLink(e){var r=e.children,t=e.resource,a="http://docs.mitmproxy.org/en/stable/"+t;return _react2.default.createElement("a",{target:"_blank",href:a},r||_react2.default.createElement("i",{className:"fa fa-question-circle"}))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=DocsLink;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes);DocsLink.propTypes={resource:_propTypes2.default.string.isRequired}; +"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function DocsLink(e){var r=e.children,t=e.resource,a="https://mitmproxy.org/docs/latest"+t;return _react2.default.createElement("a",{target:"_blank",href:a},r||_react2.default.createElement("i",{className:"fa fa-question-circle"}))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=DocsLink;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes);DocsLink.propTypes={resource:_propTypes2.default.string.isRequired}; },{"prop-types":"prop-types","react":"react"}],48:[function(require,module,exports){ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.Divider=void 0;var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames),Divider=exports.Divider=function(){return _react2.default.createElement("hr",{className:"divider"})},Dropdown=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={open:!1},n.close=n.close.bind(n),n.open=n.open.bind(n),n}return _inherits(t,e),_createClass(t,[{key:"close",value:function(){this.setState({open:!1}),document.removeEventListener("click",this.close)}},{key:"open",value:function(e){e.preventDefault(),this.state.open||(this.setState({open:!this.state.open}),document.addEventListener("click",this.close))}},{key:"render",value:function(){var e=this.props,t=e.dropup,r=e.className,n=e.btnClass,o=e.text,s=e.children;return _react2.default.createElement("div",{className:(0,_classnames2.default)(t?"dropup":"dropdown",r,{open:this.state.open})},_react2.default.createElement("a",{href:"#",className:n,onClick:this.open},o),_react2.default.createElement("ul",{className:"dropdown-menu",role:"menu"},s.map(function(e,t){return _react2.default.createElement("li",{key:t}," ",e," ")})))}}]),t}(_react.Component);Dropdown.propTypes={dropup:_propTypes2.default.bool,className:_propTypes2.default.string,btnClass:_propTypes2.default.string.isRequired},Dropdown.defaultProps={dropup:!1},exports.default=Dropdown; diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index b21ac0b8..5c02b072 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -73,11 +73,13 @@ def format_timestamp_with_milli(s): return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] -def format_address(address: tuple) -> str: +def format_address(address: typing.Optional[tuple]) -> str: """ This function accepts IPv4/IPv6 tuples and returns the formatted address string with port number """ + if address is None: + return "<no address>" try: host = ipaddress.ip_address(address[0]) if host.is_unspecified: diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 22db68f5..8ec68fa0 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -7,26 +7,17 @@ Type = typing.Union[ def sequence_type(typeinfo: typing.Type[typing.List]) -> Type: """Return the type of a sequence, e.g. typing.List""" - try: - return typeinfo.__args__[0] # type: ignore - except AttributeError: # Python 3.5.0 - return typeinfo.__parameters__[0] # type: ignore + return typeinfo.__args__[0] # type: ignore def tuple_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: """Return the types of a typing.Tuple""" - try: - return typeinfo.__args__ # type: ignore - except AttributeError: # Python 3.5.x - return typeinfo.__tuple_params__ # type: ignore + return typeinfo.__args__ # type: ignore def union_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: """return the types of a typing.Union""" - try: - return typeinfo.__args__ # type: ignore - except AttributeError: # Python 3.5.x - return typeinfo.__union_params__ # type: ignore + return typeinfo.__args__ # type: ignore def mapping_types(typeinfo: typing.Type[typing.Mapping]) -> typing.Tuple[Type, Type]: diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 66257852..253503f4 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -57,7 +57,7 @@ class WebSocketMessage(serializable.Serializable): class WebSocketFlow(flow.Flow): """ - A WebsocketFlow is a simplified representation of a Websocket connection. + A WebSocketFlow is a simplified representation of a Websocket connection. """ def __init__(self, client_conn, server_conn, handshake_flow, live=None): diff --git a/release/README.md b/release/README.md index b2f97aab..ab9992cc 100644 --- a/release/README.md +++ b/release/README.md @@ -5,9 +5,9 @@ Make sure run all these steps on the correct branch you want to create a new rel - Update CHANGELOG - Verify that all CI tests pass - Tag the release and push to Github - - For alphas, betas, and release candidates, use lightweight tags. + - For alphas, betas, and release candidates, use lightweight tags. This is necessary so that the .devXXXX counter does not reset. - - For final releases, use annotated tags. + - For final releases, use annotated tags. This makes the .devXXXX counter reset. - Wait for tag CI to complete @@ -27,7 +27,7 @@ Make sure run all these steps on the correct branch you want to create a new rel - Create a new branch based of master for major versions. - Update the dependencies in [alpine/requirements.txt](https://github.com/mitmproxy/docker-releases/commit/3d6a9989fde068ad0aea257823ac3d7986ff1613#diff-9b7e0eea8ae74688b1ac13ea080549ba) * Creating a fresh venv, pip-installing the new wheel in there, and then export all packages: - * `virtualenv -ppython3.5 venv && source venv/bin/activate && pip install mitmproxy && pip freeze` + * `virtualenv -ppython3.6 venv && source venv/bin/activate && pip install mitmproxy && pip freeze` - Tag the commit with the correct version * `2.0.0` for new major versions * `2.0.2` for new patch versions @@ -40,6 +40,11 @@ Make sure run all these steps on the correct branch you want to create a new rel - `git push --tags` to push the new tag - Check the build details page again +## Website + - Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml + - `./build && ./upload-test` + - If everything looks alright: `./upload-prod` + ## Prepare for next release - Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases. diff --git a/release/ci.py b/release/ci.py new file mode 100755 index 00000000..94b1f13d --- /dev/null +++ b/release/ci.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 + +import contextlib +import os +import platform +import sys +import shutil +import subprocess +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename + +import cryptography.fernet +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] +else: + VENV_BIN = "bin" + PYINSTALLER_ARGS = [] + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") + +BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +# PyInstaller 3.2 does not bundle pydivert's Windivert binaries +PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + +VENV_DIR = join(BUILD_DIR, "venv") + +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] + +TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) +BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) +if TAG: + VERSION = TAG + UPLOAD_DIR = VERSION +elif BRANCH: + VERSION = BRANCH + UPLOAD_DIR = "branches/%s" % VERSION +else: + print("Could not establish build name - exiting." % BRANCH) + sys.exit(0) + + +print("BUILD VERSION=%s" % VERSION) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=VERSION, + platform=PLATFORM_TAG, + ext=ext + ) + + +def wheel_name() -> str: + return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) + + +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=VERSION, + platform=PLATFORM_TAG, + ext=ext, + ) + + +@contextlib.contextmanager +def chdir(path: str): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("info") +def info(): + print("Version: %s" % VERSION) + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + os.makedirs(DIST_DIR, exist_ok=True) + + if "WHEEL" in os.environ: + print("Building wheel...") + subprocess.check_call( + [ + "python", + "setup.py", "-q", "bdist_wheel", + "--dist-dir", "release/dist", + ] + ) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + print("> %s --version" % executable) + print(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) + + +def is_pr(): + if "TRAVIS_PULL_REQUEST" in os.environ: + if os.environ["TRAVIS_PULL_REQUEST"] == "false": + return False + return True + elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"): + return True + return False + + +@cli.command("upload") +def upload(): + """ + Upload snapshot to snapshot server + """ + # This requires some explanation. The AWS access keys are only exposed to + # privileged builds - that is, they are not available to PRs from forks. + # However, they ARE exposed to PRs from a branch within the main repo. This + # check catches that corner case, and prevents an inadvertent upload. + if is_pr(): + print("Refusing to upload a pull request") + return + if "AWS_ACCESS_KEY_ID" in os.environ: + subprocess.check_call( + [ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, + "--recursive", + ] + ) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + +if __name__ == "__main__": + cli() diff --git a/release/known_hosts.enc b/release/known_hosts.enc deleted file mode 100644 index 585ee678..00000000 --- a/release/known_hosts.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7
\ No newline at end of file diff --git a/release/rtool.pem b/release/rtool.pem deleted file mode 100644 index 097dff1a..00000000 --- a/release/rtool.pem +++ /dev/null @@ -1,68 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB - -0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y -byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU -V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z -T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh -PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl -w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD -P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt -JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE -yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km -cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe -hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8 -jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx -QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey -OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL -kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR -AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc -sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko -SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz -5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P -cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk -kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/ -Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9 -3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4 -mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t -8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j -FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf -EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP -a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR -AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+ -m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P -bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q -QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx -uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq -yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K -u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK -MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0 -8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq -y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz -G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi -o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq -2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL -5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp -30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac -7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA -1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8 -eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB -bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO -VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+ -2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175 ------END RSA PRIVATE KEY----- ------BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y -qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe -24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX -FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ -0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI -KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt -0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+ -tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ -TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ -q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J -6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD -5I1jQDXFbjAgAnaSyqyW368CAwEAAQ== ------END PUBLIC KEY-----
\ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 25f37e61..0009cf89 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,79 +1,22 @@ #!/usr/bin/env python3 import contextlib -import fnmatch import os import sys import platform -import re import runpy import shlex -import shutil import subprocess -import tarfile -import zipfile -from os.path import join, abspath, dirname, exists, basename +from os.path import join, abspath, dirname -import click import cryptography.fernet -import pysftp - -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if platform.system() == "Windows": - VENV_BIN = "Scripts" - PYINSTALLER_ARGS = [ - # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. - "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", - ] -else: - VENV_BIN = "bin" - PYINSTALLER_ARGS = [] - -# ZipFile and tarfile have slightly different APIs. Fix that. -if platform.system() == "Windows": - def Archive(name): - a = zipfile.ZipFile(name, "w") - a.add = a.write - return a -else: - def Archive(name): - return tarfile.open(name, "w:gz") +import click -PLATFORM_TAG = { - "Darwin": "osx", - "Windows": "windows", - "Linux": "linux", -}.get(platform.system(), platform.system()) ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") - -BUILD_DIR = join(RELEASE_DIR, "build") DIST_DIR = join(RELEASE_DIR, "dist") - -PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") -# PyInstaller 3.2 does not bundle pydivert's Windivert binaries -PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") -PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") -PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) - -VENV_DIR = join(BUILD_DIR, "venv") - -# Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") -BDISTS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] -} -if platform.system() == "Windows": - BDISTS["mitmproxy"].remove("mitmproxy") - -TOOLS = [ - tool - for tools in sorted(BDISTS.values()) - for tool in tools -] def git(args: str) -> str: @@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str: return x["get_version"](dev, build, True) -def set_version(dev: bool) -> None: - """ - Update version information in mitmproxy's version.py to either include hardcoded information or not. - """ - version = get_version(dev, dev) - with open(VERSION_FILE, "r") as f: - content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) - with open(VERSION_FILE, "w") as f: - f.write(content) - - -def archive_name(bdist: str) -> str: - if platform.system() == "Windows": - ext = "zip" - else: - ext = "tar.gz" - return "{project}-{version}-{platform}.{ext}".format( - project=bdist, - version=get_version(), - platform=PLATFORM_TAG, - ext=ext - ) - - def wheel_name() -> str: return "mitmproxy-{version}-py3-none-any.whl".format( version=get_version(True), ) -def installer_name() -> str: - ext = { - "Windows": "exe", - "Darwin": "dmg", - "Linux": "run" - }[platform.system()] - return "mitmproxy-{version}-{platform}-installer.{ext}".format( - version=get_version(), - platform=PLATFORM_TAG, - ext=ext, - ) - - @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -146,24 +51,6 @@ def cli(): pass -@cli.command("encrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def encrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.encrypt(infile.read())) - - -@cli.command("decrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.decrypt(infile.read())) - - @cli.command("contributors") def contributors(): """ @@ -176,97 +63,6 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("wheel") -def make_wheel(): - """ - Build a Python wheel - """ - set_version(True) - try: - subprocess.check_call([ - "tox", "-e", "wheel", - ], env={ - **os.environ, - "VERSION": get_version(True), - }) - finally: - set_version(False) - - -@cli.command("bdist") -def make_bdist(): - """ - Build a binary distribution - """ - if exists(PYINSTALLER_TEMP): - shutil.rmtree(PYINSTALLER_TEMP) - if exists(PYINSTALLER_DIST): - shutil.rmtree(PYINSTALLER_DIST) - - os.makedirs(DIST_DIR, exist_ok=True) - - for bdist, tools in sorted(BDISTS.items()): - with Archive(join(DIST_DIR, archive_name(bdist))) as archive: - for tool in tools: - # We can't have a folder and a file with the same name. - if tool == "mitmproxy": - tool = "mitmproxy_main" - # This is PyInstaller, so it messes up paths. - # We need to make sure that we are in the spec folder. - with chdir(PYINSTALLER_SPEC): - print("Building %s binary..." % tool) - excludes = [] - if tool != "mitmweb": - excludes.append("mitmproxy.tools.web") - if tool != "mitmproxy_main": - excludes.append("mitmproxy.tools.console") - - # Overwrite mitmproxy/version.py to include commit info - set_version(True) - try: - subprocess.check_call( - [ - "pyinstaller", - "--clean", - "--workpath", PYINSTALLER_TEMP, - "--distpath", PYINSTALLER_DIST, - "--additional-hooks-dir", PYINSTALLER_HOOKS, - "--onefile", - "--console", - "--icon", "icon.ico", - # This is PyInstaller, so setting a - # different log level obviously breaks it :-) - # "--log-level", "WARN", - ] - + [x for e in excludes for x in ["--exclude-module", e]] - + PYINSTALLER_ARGS - + [tool] - ) - finally: - set_version(False) - # Delete the spec file - we're good without. - os.remove("{}.spec".format(tool)) - - # Test if it works at all O:-) - executable = join(PYINSTALLER_DIST, tool) - if platform.system() == "Windows": - executable += ".exe" - - # Remove _main suffix from mitmproxy executable - if "_main" in executable: - shutil.move( - executable, - executable.replace("_main", "") - ) - executable = executable.replace("_main", "") - - print("> %s --version" % executable) - print(subprocess.check_output([executable, "--version"]).decode()) - - archive.add(executable, basename(executable)) - print("Packed {}.".format(archive_name(bdist))) - - @cli.command("upload-release") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @@ -305,77 +101,13 @@ def homebrew_pr(): ]) -@cli.command("upload-snapshot") -@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) -@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) -@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) -@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) -@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) -@click.option("--wheel/--no-wheel", default=False) -@click.option("--bdist/--no-bdist", default=False) -@click.option("--installer/--no-installer", default=False) -def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): - """ - Upload snapshot to snapshot server - """ - cnopts = pysftp.CnOpts( - knownhosts=join(RELEASE_DIR, 'known_hosts') - ) - with pysftp.Connection(host=host, - port=port, - username=user, - private_key=private_key, - private_key_pass=private_key_password, - cnopts=cnopts) as sftp: - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - if bdist: - for bdist in sorted(BDISTS.keys()): - files.append(archive_name(bdist)) - if installer: - files.append(installer_name()) - - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = re.sub( - r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), - get_version(True, True), - f - ) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - # We hide the file during upload - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Show new version - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - if f != wheel_name(): - # "latest" isn't a proper wheel version, so this could not be installed. - # https://github.com/mitmproxy/mitmproxy/issues/1065 - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) if __name__ == "__main__": @@ -35,7 +35,6 @@ setup( "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", @@ -65,7 +64,7 @@ setup( "brotlipy>=0.7.0,<0.8", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", - "cryptography>=2.1.4,<2.2", + "cryptography>=2.1.4,<2.3", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", @@ -77,7 +76,7 @@ setup( "pyperclip>=1.6.0, <1.7", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4, <1.6", - "tornado>=4.3, <4.6", + "tornado>=4.3,<5.1", "urwid>=2.0.1,<2.1", "wsproto>=0.11.0,<0.12.0", ], @@ -88,14 +87,14 @@ setup( 'dev': [ "flake8>=3.5, <3.6", "Flask>=0.10.1, <0.13", - "mypy>=0.560,<0.561", + "mypy>=0.580,<0.581", "pytest-cov>=2.5.1,<3", "pytest-faulthandler>=1.3.1,<2", "pytest-timeout>=1.2.1,<2", "pytest-xdist>=1.22,<2", "pytest>=3.3,<4", "requests>=2.9.1, <3", - "tox>=2.3, <3", + "tox>=3.0,<3.1", "rstcheck>=2.2, <4.0", ], 'examples': [ diff --git a/test/bench/.gitignore b/test/bench/.gitignore new file mode 100644 index 00000000..1a06816d --- /dev/null +++ b/test/bench/.gitignore @@ -0,0 +1 @@ +results diff --git a/test/bench/README.md b/test/bench/README.md new file mode 100644 index 00000000..05741c07 --- /dev/null +++ b/test/bench/README.md @@ -0,0 +1,56 @@ + +This directory contains a set of tools for benchmarking and profiling mitmproxy. +At the moment, this is simply to give developers a quick way to see the impact +of their work. Eventually, this might grow into a performance dashboard with +historical data, so we can track performance over time. + + +# Setup + +Install the following tools: + + go get -u github.com/rakyll/hey + go get github.com/cortesi/devd/cmd/devd + +You may also want to install snakeviz to make viewing profiles easier: + + pip install snakeviz + +In one window, run the devd server: + + ./backend + + +# Running tests + +Each run consists of two files - a mitproxy invocation, and a traffic generator. +Make sure the backend is started, then run the proxy: + + ./simple.mitmproxy + +Now run the traffic generator: + + ./simple.traffic + +After the run is done, quit the proxy with ctrl-c. + + +# Reading results + +Results are placed in the ./results directory. You should see two files - a +performance log from **hey**, and a profile. You can view the profile like so: + + snakeviz ./results/simple.prof + + + + + + + + + + + + + diff --git a/test/bench/backend b/test/bench/backend new file mode 100755 index 00000000..12a05d70 --- /dev/null +++ b/test/bench/backend @@ -0,0 +1,3 @@ +#!/bin/sh + +devd -p 10001 .
\ No newline at end of file diff --git a/test/bench/profiler.py b/test/bench/profiler.py new file mode 100644 index 00000000..9072e17d --- /dev/null +++ b/test/bench/profiler.py @@ -0,0 +1,25 @@ +import cProfile +from mitmproxy import ctx + + +class Profile: + """ + A simple profiler addon. + """ + def __init__(self): + self.pr = cProfile.Profile() + + def load(self, loader): + loader.add_option( + "profile_path", + str, + "/tmp/profile", + "Destination for the run profile, saved at exit" + ) + self.pr.enable() + + def done(self): + self.pr.dump_stats(ctx.options.profile_path) + + +addons = [Profile()]
\ No newline at end of file diff --git a/test/bench/simple.mitmproxy b/test/bench/simple.mitmproxy new file mode 100755 index 00000000..9de32981 --- /dev/null +++ b/test/bench/simple.mitmproxy @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p results +mitmdump -p 10002 --mode reverse:http://devd.io:10001 \ + -s ./profiler.py --set profile_path=./results/simple.prof diff --git a/test/bench/simple.traffic b/test/bench/simple.traffic new file mode 100755 index 00000000..08200e05 --- /dev/null +++ b/test/bench/simple.traffic @@ -0,0 +1,3 @@ +#!/bin/sh + +hey -disable-keepalive http://localhost:10002/profiler.py | tee ./results/simple.perf
\ No newline at end of file diff --git a/test/filename_matching.py b/test/filename_matching.py index e74848d4..5f49725e 100644..100755 --- a/test/filename_matching.py +++ b/test/filename_matching.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import os import re import glob diff --git a/test/individual_coverage.py b/test/individual_coverage.py index c975b4c8..097b290f 100644..100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import io import contextlib import os diff --git a/test/mitmproxy/addons/test_onboarding.py b/test/mitmproxy/addons/test_onboarding.py index 810ddef1..0d99b1ff 100644 --- a/test/mitmproxy/addons/test_onboarding.py +++ b/test/mitmproxy/addons/test_onboarding.py @@ -4,6 +4,10 @@ from mitmproxy.addons import onboarding from mitmproxy.test import taddons from .. import tservers +import asyncio +import tornado.platform.asyncio +asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy()) + class TestApp(tservers.HTTPProxyTest): def addons(self): diff --git a/test/mitmproxy/data/addonscripts/shutdown.py b/test/mitmproxy/data/addonscripts/shutdown.py index 51a99b5c..3da4d03e 100644 --- a/test/mitmproxy/data/addonscripts/shutdown.py +++ b/test/mitmproxy/data/addonscripts/shutdown.py @@ -1,5 +1,5 @@ from mitmproxy import ctx -def running(): +def tick(): ctx.master.shutdown() diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index ce49002c..ef581a91 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -351,10 +351,10 @@ class TestRequestUtils: request.headers["Content-Type"] = "application/x-www-form-urlencoded" assert list(request.urlencoded_form.items()) == [("foobar", "baz")] request.raw_content = b"\xFF" - assert len(request.urlencoded_form) == 0 + assert len(request.urlencoded_form) == 1 def test_set_urlencoded_form(self): - request = treq() + request = treq(content=b"\xec\xed") request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')] assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" assert request.content diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index d9aa03b4..13f28728 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -10,7 +10,6 @@ import h2 from mitmproxy import options import mitmproxy.net -from mitmproxy.addons import core from ...net import tservers as net_tservers from mitmproxy import exceptions from mitmproxy.net.http import http1, http2 @@ -90,9 +89,7 @@ class _Http2TestBase: @classmethod def setup_class(cls): cls.options = cls.get_options() - tmaster = tservers.TestMaster(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() @classmethod @@ -120,6 +117,7 @@ class _Http2TestBase: def teardown(self): if self.client: self.client.close() + self.server.server.wait_for_silence() def setup_connection(self): self.client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 661605b7..e5ed8e9d 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -3,10 +3,10 @@ import os import struct import tempfile import traceback +import time from mitmproxy import options from mitmproxy import exceptions -from mitmproxy.addons import core from mitmproxy.http import HTTPFlow from mitmproxy.websocket import WebSocketFlow @@ -52,9 +52,7 @@ class _WebSocketTestBase: @classmethod def setup_class(cls): cls.options = cls.get_options() - tmaster = tservers.TestMaster(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() @classmethod @@ -163,7 +161,7 @@ class TestSimple(_WebSocketTest): def websocket_start(self, f): f.stream = streaming - self.master.addons.add(Stream()) + self.proxy.set_addons(Stream()) self.setup_connection() frame = websockets.Frame.from_file(self.client.rfile) @@ -204,7 +202,7 @@ class TestSimple(_WebSocketTest): def websocket_message(self, f): f.messages[-1].content = "foo" - self.master.addons.add(Addon()) + self.proxy.set_addons(Addon()) self.setup_connection() frame = websockets.Frame.from_file(self.client.rfile) @@ -235,7 +233,7 @@ class TestKillFlow(_WebSocketTest): def websocket_message(self, f): f.kill() - self.master.addons.add(KillFlow()) + self.proxy.set_addons(KillFlow()) self.setup_connection() with pytest.raises(exceptions.TcpDisconnect): @@ -329,7 +327,12 @@ class TestPong(_WebSocketTest): assert frame.header.opcode == websockets.OPCODE.PONG assert frame.payload == b'foobar' - assert self.master.has_log("Pong Received from server", "info") + for i in range(20): + if self.master.has_log("Pong Received from server", "info"): + break + time.sleep(0.01) + else: + raise AssertionError("No pong seen") class TestClose(_WebSocketTest): @@ -405,7 +408,7 @@ class TestStreaming(_WebSocketTest): def websocket_start(self, f): f.stream = streaming - self.master.addons.add(Stream()) + self.proxy.set_addons(Stream()) self.setup_connection() frame = None diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 986dfb39..f594fb40 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -21,14 +21,6 @@ from pathod import pathod from .. import tservers from ...conftest import skip_appveyor -""" - Note that the choice of response code in these tests matters more than you - might think. libcurl treats a 304 response code differently from, say, a - 200 response code - it will correctly terminate a 304 response with no - content-length header, whereas it will block forever waiting for content - for a 200 response. -""" - class CommonMixin: @@ -237,28 +229,14 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): p.request("get:'%s'" % response) def test_reconnect(self): - req = "get:'%s/p/200:b@1'" % self.server.urlbase + req = "get:'%s/p/200:b@1:da'" % self.server.urlbase p = self.pathoc() - class MockOnce: - call = 0 - - def mock_once(self, http1obj, req): - self.call += 1 - if self.call == 1: - raise exceptions.TcpDisconnect - else: - headers = http1.assemble_request_head(req) - http1obj.server_conn.wfile.write(headers) - http1obj.server_conn.wfile.flush() - with p.connect(): - with mock.patch("mitmproxy.proxy.protocol.http1.Http1Layer.send_request_headers", - side_effect=MockOnce().mock_once, autospec=True): - # Server disconnects while sending headers but mitmproxy reconnects - resp = p.request(req) - assert resp - assert resp.status_code == 200 + assert p.request(req) + # Server has disconnected. Mitmproxy should detect this, and reconnect. + assert p.request(req) + assert p.request(req) def test_get_connection_switching(self): req = "get:'%s/p/200:b@1'" @@ -284,10 +262,9 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): s = script.Script( tutils.test_data.path("mitmproxy/data/addonscripts/stream_modify.py") ) - self.master.addons.add(s) + self.set_addons(s) d = self.pathod('200:b"foo"') assert d.content == b"bar" - self.master.addons.remove(s) def test_first_line_rewrite(self): """ @@ -591,12 +568,11 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): s = script.Script( tutils.test_data.path("mitmproxy/data/addonscripts/tcp_stream_modify.py") ) - self.master.addons.add(s) + self.set_addons(s) self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() assert d.content == b"bar" - self.master.addons.remove(s) class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): @@ -739,7 +715,7 @@ class TestRedirectRequest(tservers.HTTPProxyTest): This test verifies that the original destination is restored for the third request. """ - self.proxy.tmaster.addons.add(ARedirectRequest(self.server2.port)) + self.set_addons(ARedirectRequest(self.server2.port)) p = self.pathoc() with p.connect(): @@ -778,7 +754,7 @@ class AStreamRequest: class TestStreamRequest(tservers.HTTPProxyTest): def test_stream_simple(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) p = self.pathoc() with p.connect(): # a request with 100k of data but without content-length @@ -787,7 +763,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert len(r1.content) > 100000 def test_stream_multiple(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) p = self.pathoc() with p.connect(): # simple request with streaming turned on @@ -799,7 +775,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert r1.status_code == 201 def test_stream_chunked(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("127.0.0.1", self.proxy.port)) fconn = connection.makefile("rb") @@ -828,7 +804,7 @@ class AFakeResponse: class TestFakeResponse(tservers.HTTPProxyTest): def test_fake(self): - self.proxy.tmaster.addons.add(AFakeResponse()) + self.set_addons(AFakeResponse()) f = self.pathod("200") assert "header-response" in f.headers @@ -844,7 +820,7 @@ class TestServerConnect(tservers.HTTPProxyTest): def test_unnecessary_serverconnect(self): """A replayed/fake response with no upstream_cert should not connect to an upstream server""" - self.proxy.tmaster.addons.add(AFakeResponse()) + self.set_addons(AFakeResponse()) assert self.pathod("200").status_code == 200 assert not self.proxy.tmaster.has_log("serverconnect") @@ -857,7 +833,7 @@ class AKillRequest: class TestKillRequest(tservers.HTTPProxyTest): def test_kill(self): - self.proxy.tmaster.addons.add(AKillRequest()) + self.set_addons(AKillRequest()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # Nothing should have hit the server @@ -871,7 +847,7 @@ class AKillResponse: class TestKillResponse(tservers.HTTPProxyTest): def test_kill(self): - self.proxy.tmaster.addons.add(AKillResponse()) + self.set_addons(AKillResponse()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # The server should have seen a request @@ -894,7 +870,7 @@ class AIncomplete: class TestIncompleteResponse(tservers.HTTPProxyTest): def test_incomplete(self): - self.proxy.tmaster.addons.add(AIncomplete()) + self.set_addons(AIncomplete()) assert self.pathod("200").status_code == 502 @@ -977,7 +953,7 @@ class TestUpstreamProxySSL( def test_change_upstream_proxy_connect(self): # skip chain[0]. - self.proxy.tmaster.addons.add( + self.set_addons( UpstreamProxyChanger( ("127.0.0.1", self.chain[1].port) ) @@ -996,8 +972,8 @@ class TestUpstreamProxySSL( Client <- HTTPS -> Proxy <- HTTP -> Proxy <- HTTPS -> Server """ - self.proxy.tmaster.addons.add(RewriteToHttp()) - self.chain[1].tmaster.addons.add(RewriteToHttps()) + self.set_addons(RewriteToHttp()) + self.chain[1].set_addons(RewriteToHttps()) p = self.pathoc() with p.connect(): resp = p.request("get:'/p/418'") @@ -1055,24 +1031,8 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): request again. """ - class MockOnce: - call = 0 - - def mock_once(self, http1obj, req): - self.call += 1 - - if self.call == 2: - headers = http1.assemble_request_head(req) - http1obj.server_conn.wfile.write(headers) - http1obj.server_conn.wfile.flush() - raise exceptions.TcpDisconnect - else: - headers = http1.assemble_request_head(req) - http1obj.server_conn.wfile.write(headers) - http1obj.server_conn.wfile.flush() - - self.chain[0].tmaster.addons.add(RequestKiller([1, 2])) - self.chain[1].tmaster.addons.add(RequestKiller([1])) + self.chain[0].set_addons(RequestKiller([1, 2])) + self.chain[1].set_addons(RequestKiller([1])) p = self.pathoc() with p.connect(): @@ -1085,9 +1045,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): assert len(self.chain[0].tmaster.state.flows) == 1 assert len(self.chain[1].tmaster.state.flows) == 1 - with mock.patch("mitmproxy.proxy.protocol.http1.Http1Layer.send_request_headers", - side_effect=MockOnce().mock_once, autospec=True): - req = p.request("get:'/p/418:b\"content2\"'") + req = p.request("get:'/p/418:b\"content2\"'") assert req.status_code == 502 diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py index e2b80753..3d0a43f8 100644 --- a/test/mitmproxy/test_command.py +++ b/test/mitmproxy/test_command.py @@ -309,6 +309,31 @@ class TDec: pass +class TAttr: + def __getattr__(self, item): + raise IOError + + +class TCmds(TAttr): + def __init__(self): + self.TAttr = TAttr() + + @command.command("empty") + def empty(self) -> None: + pass + + +def test_collect_commands(): + """ + This tests for the error thrown by hasattr() + """ + with taddons.context() as tctx: + c = command.CommandManager(tctx.master) + a = TCmds() + c.collect_commands(a) + assert "empty" in c.commands + + def test_decorator(): with taddons.context() as tctx: c = command.CommandManager(tctx.master) diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 00cdbc87..845a9043 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -38,6 +38,9 @@ class TestClientConnection: assert 'ALPN' not in repr(c) assert 'TLS' in repr(c) + c.address = None + assert repr(c) + def test_tls_established_property(self): c = tflow.tclient_conn() c.tls_established = True @@ -110,6 +113,9 @@ class TestServerConnection: c.tls_established = False assert 'TLS' not in repr(c) + c.address = None + assert repr(c) + def test_tls_established_property(self): c = tflow.tserver_conn() c.tls_established = True diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index e840380a..f7c64ed9 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -1,82 +1,31 @@ -from threading import Thread, Event -from unittest.mock import Mock +import asyncio import queue import pytest from mitmproxy.exceptions import Kill, ControlException from mitmproxy import controller -from mitmproxy import master -from mitmproxy import proxy from mitmproxy.test import taddons -class TMsg: - pass +@pytest.mark.asyncio +async def test_master(): + class TMsg: + pass + class tAddon: + def log(self, _): + ctx.master.should_exit.set() -class TestMaster: - def test_simple(self): - class tAddon: - def log(self, _): - ctx.master.should_exit.set() + with taddons.context(tAddon()) as ctx: + assert not ctx.master.should_exit.is_set() - with taddons.context() as ctx: - ctx.master.addons.add(tAddon()) - assert not ctx.master.should_exit.is_set() + async def test(): msg = TMsg() msg.reply = controller.DummyReply() - ctx.master.event_queue.put(("log", msg)) - ctx.master.run() - assert ctx.master.should_exit.is_set() - - def test_server_simple(self): - m = master.Master(None) - m.server = proxy.DummyServer() - m.start() - m.shutdown() - m.start() - m.shutdown() + await ctx.master.channel.tell("log", msg) - -class TestServerThread: - def test_simple(self): - m = Mock() - t = master.ServerThread(m) - t.run() - assert m.serve_forever.called - - -class TestChannel: - def test_tell(self): - q = queue.Queue() - channel = controller.Channel(q, Event()) - m = Mock(name="test_tell") - channel.tell("test", m) - assert q.get() == ("test", m) - assert m.reply - - def test_ask_simple(self): - q = queue.Queue() - - def reply(): - m, obj = q.get() - assert m == "test" - obj.reply.send(42) - obj.reply.take() - obj.reply.commit() - - Thread(target=reply).start() - - channel = controller.Channel(q, Event()) - assert channel.ask("test", Mock(name="test_ask_simple")) == 42 - - def test_ask_shutdown(self): - q = queue.Queue() - done = Event() - done.set() - channel = controller.Channel(q, done) - with pytest.raises(Kill): - channel.ask("test", Mock(name="test_ask_shutdown")) + asyncio.ensure_future(test()) + assert not ctx.master.should_exit.is_set() class TestReply: diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 8cc11a16..4042de5b 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -7,7 +7,7 @@ import mitmproxy.io from mitmproxy import flowfilter from mitmproxy import options from mitmproxy.io import tnetstring -from mitmproxy.exceptions import FlowReadException, ReplayException, ControlException +from mitmproxy.exceptions import FlowReadException, ReplayException from mitmproxy import flow from mitmproxy import http from mitmproxy.net import http as net_http @@ -169,9 +169,10 @@ class TestFlowMaster: f.error = flow.Error("msg") fm.addons.handle_lifecycle("error", f) - fm.tell("foo", f) - with pytest.raises(ControlException): - fm.tick(timeout=1) + # FIXME: This no longer works, because we consume on the main loop. + # fm.tell("foo", f) + # with pytest.raises(ControlException): + # fm.addons.trigger("unknown") fm.shutdown() diff --git a/test/mitmproxy/test_fuzzing.py b/test/mitmproxy/test_fuzzing.py index 905ba1cd..57d0ca55 100644 --- a/test/mitmproxy/test_fuzzing.py +++ b/test/mitmproxy/test_fuzzing.py @@ -25,14 +25,4 @@ class TestFuzzy(tservers.HTTPProxyTest): p = self.pathoc() with p.connect(): resp = p.request(req % self.server.port) - assert resp.status_code == 400 - - # def test_invalid_upstream(self): - # req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'" - # p = self.pathoc() - # assert p.request(req % self.server.port).status_code == 502 - - # def test_upstream_disconnect(self): - # req = r'200:d0' - # p = self.pathod(req) - # assert p.status_code == 502 + assert resp.status_code == 400
\ No newline at end of file diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py index db8a63a7..108f238e 100644 --- a/test/mitmproxy/tools/console/test_statusbar.py +++ b/test/mitmproxy/tools/console/test_statusbar.py @@ -1,3 +1,5 @@ +import pytest + from mitmproxy import options from mitmproxy.tools.console import statusbar, master @@ -31,3 +33,29 @@ def test_statusbar(monkeypatch): bar = statusbar.StatusBar(m) # this already causes a redraw assert bar.ib._w + + +@pytest.mark.parametrize("message,ready_message", [ + ("", [(None, ""), ("warn", "")]), + (("info", "Line fits into statusbar"), [("info", "Line fits into statusbar"), + ("warn", "")]), + ("Line doesn't fit into statusbar", [(None, "Line doesn'\u2026"), + ("warn", "(more in eventlog)")]), + (("alert", "Two lines.\nFirst fits"), [("alert", "Two lines."), + ("warn", "(more in eventlog)")]), + ("Two long lines\nFirst doesn't fit", [(None, "Two long li\u2026"), + ("warn", "(more in eventlog)")]) +]) +def test_shorten_message(message, ready_message): + o = options.Options() + m = master.ConsoleMaster(o) + ab = statusbar.ActionBar(m) + assert ab.shorten_message(message, max_width=30) == ready_message + + +def test_shorten_message_narrow(): + o = options.Options() + m = master.ConsoleMaster(o) + ab = statusbar.ActionBar(m) + shorten_msg = ab.shorten_message("error", max_width=4) + assert shorten_msg == [(None, "\u2026"), ("warn", "(more in eventlog)")] diff --git a/test/mitmproxy/tools/test_main.py b/test/mitmproxy/tools/test_main.py index 88e2fe86..a514df74 100644 --- a/test/mitmproxy/tools/test_main.py +++ b/test/mitmproxy/tools/test_main.py @@ -1,19 +1,25 @@ +import pytest + from mitmproxy.test import tutils from mitmproxy.tools import main shutdown_script = tutils.test_data.path("mitmproxy/data/addonscripts/shutdown.py") -def test_mitmweb(): +@pytest.mark.asyncio +async def test_mitmweb(): main.mitmweb([ "--no-web-open-browser", "-q", + "-p", "0", "-s", shutdown_script ]) -def test_mitmdump(): +@pytest.mark.asyncio +async def test_mitmdump(): main.mitmdump([ "-q", + "-p", "0", "-s", shutdown_script ]) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 0040b023..2d102a5d 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -2,7 +2,9 @@ import os.path import threading import tempfile import sys +import time from unittest import mock +import asyncio import mitmproxy.platform from mitmproxy.addons import core @@ -12,6 +14,7 @@ from mitmproxy import controller from mitmproxy import options from mitmproxy import exceptions from mitmproxy import io +from mitmproxy.utils import human import pathod.test import pathod.pathoc @@ -62,11 +65,6 @@ class TestState: if f not in self.flows: self.flows.append(f) - # TODO: add TCP support? - # def tcp_start(self, f): - # if f not in self.flows: - # self.flows.append(f) - class TestMaster(taddons.RecordingMaster): @@ -90,13 +88,12 @@ class TestMaster(taddons.RecordingMaster): class ProxyThread(threading.Thread): - def __init__(self, tmaster): + def __init__(self, masterclass, options): threading.Thread.__init__(self) - self.tmaster = tmaster - self.name = "ProxyThread (%s:%s)" % ( - tmaster.server.address[0], - tmaster.server.address[1], - ) + self.masterclass = masterclass + self.options = options + self.tmaster = None + self.event_loop = None controller.should_exit = False @property @@ -107,11 +104,27 @@ class ProxyThread(threading.Thread): def tlog(self): return self.tmaster.logs + def shutdown(self): + self.tmaster.shutdown() + def run(self): + self.event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.event_loop) + self.tmaster = self.masterclass(self.options) + self.tmaster.addons.add(core.Core()) + self.name = "ProxyThread (%s)" % human.format_address(self.tmaster.server.address) self.tmaster.run() - def shutdown(self): - self.tmaster.shutdown() + def set_addons(self, *addons): + self.tmaster.reset(addons) + self.tmaster.addons.trigger("tick") + + def start(self): + super().start() + while True: + if self.tmaster: + break + time.sleep(0.01) class ProxyTestBase: @@ -132,9 +145,7 @@ class ProxyTestBase: ssloptions=cls.ssloptions) cls.options = cls.get_options() - tmaster = cls.masterclass(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = ProxyThread(tmaster) + cls.proxy = ProxyThread(cls.masterclass, cls.options) cls.proxy.start() @classmethod @@ -173,6 +184,9 @@ class ProxyTestBase: ssl_insecure=True, ) + def set_addons(self, *addons): + self.proxy.set_addons(*addons) + def addons(self): """ Can be over-ridden to add a standard set of addons to tests. @@ -327,8 +341,7 @@ class SocksModeTest(HTTPProxyTest): return opts -class ChainProxyTest(ProxyTestBase): - +class HTTPUpstreamProxyTest(HTTPProxyTest): """ Chain three instances of mitmproxy in a row to test upstream mode. Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1] @@ -344,11 +357,12 @@ class ChainProxyTest(ProxyTestBase): cls.chain = [] for _ in range(cls.n): opts = cls.get_options() - tmaster = cls.masterclass(opts) - tmaster.addons.add(core.Core()) - proxy = ProxyThread(tmaster) + proxy = ProxyThread(cls.masterclass, opts) proxy.start() cls.chain.insert(0, proxy) + while True: + if proxy.event_loop and proxy.event_loop.is_running(): + break super().setup_class() @@ -372,7 +386,3 @@ class ChainProxyTest(ProxyTestBase): mode="upstream:" + s, ) return opts - - -class HTTPUpstreamProxyTest(ChainProxyTest, HTTPProxyTest): - pass diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index 947cfa4a..faf35f72 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -56,3 +56,4 @@ def test_format_address(): assert human.format_address(("example.com", "54010")) == "example.com:54010" assert human.format_address(("::", "8080")) == "*:8080" assert human.format_address(("0.0.0.0", "8080")) == "*:8080" + assert human.format_address(None) == "<no address>" diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 9cb4334e..85713e14 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -1,6 +1,5 @@ import io import typing -from unittest import mock import pytest from mitmproxy.utils import typecheck @@ -32,12 +31,6 @@ def test_check_union(): with pytest.raises(TypeError): typecheck.check_option_type("foo", [], typing.Union[int, str]) - # Python 3.5 only defines __union_params__ - m = mock.Mock() - m.__str__ = lambda self: "typing.Union" - m.__union_params__ = (int,) - typecheck.check_option_type("foo", 42, m) - def test_check_tuple(): typecheck.check_option_type("foo", (42, "42"), typing.Tuple[int, str]) @@ -50,12 +43,6 @@ def test_check_tuple(): with pytest.raises(TypeError): typecheck.check_option_type("foo", ("42", 42), typing.Tuple[int, str]) - # Python 3.5 only defines __tuple_params__ - m = mock.Mock() - m.__str__ = lambda self: "typing.Tuple" - m.__tuple_params__ = (int, str) - typecheck.check_option_type("foo", (42, "42"), m) - def test_check_sequence(): typecheck.check_option_type("foo", [10], typing.Sequence[int]) @@ -68,12 +55,6 @@ def test_check_sequence(): with pytest.raises(TypeError): typecheck.check_option_type("foo", "foo", typing.Sequence[str]) - # Python 3.5 only defines __parameters__ - m = mock.Mock() - m.__str__ = lambda self: "typing.Sequence" - m.__parameters__ = (int,) - typecheck.check_option_type("foo", [10], m) - def test_check_io(): typecheck.check_option_type("foo", io.StringIO(), typing.IO[str]) @@ -1,5 +1,5 @@ [tox] -envlist = py35, py36, lint +envlist = py36, lint skipsdist = True toxworkdir={env:TOX_WORK_DIR:.tox} @@ -21,7 +21,7 @@ commands = commands = mitmdump --version flake8 --jobs 8 mitmproxy pathod examples test release - python test/filename_matching.py + python ./test/filename_matching.py rstcheck README.rst mypy --ignore-missing-imports ./mitmproxy ./pathod mypy --ignore-missing-imports --follow-imports=skip ./examples/simple/ ./examples/pathod/ ./examples/complex/ @@ -30,29 +30,23 @@ commands = deps = -rrequirements.txt commands = - python test/individual_coverage.py + python ./test/individual_coverage.py -[testenv:wheel] -recreate = True +[testenv:cibuild] +passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps = + -rrequirements.txt + pyinstaller==3.3.1 + awscli commands = - python setup.py -q bdist_wheel --dist-dir release/dist - pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl - # skip `mitmproxy --version` if SKIP_MITMPROXY is defined. - {env:SKIP_MITMPROXY:mitmproxy --version} mitmdump --version - mitmweb --version - pathod --version - pathoc --version + python ./release/ci.py {posargs} -[testenv:rtool] -passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY +[testenv:docs] +passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps = -rrequirements.txt - pyinstaller==3.3.1 - twine==1.9.1 - pysftp==0.2.9 - + awscli +changedir = docs commands = - mitmdump --version - python ./release/rtool.py {posargs} + ./ci diff --git a/web/README b/web/README deleted file mode 100644 index c8e60379..00000000 --- a/web/README +++ /dev/null @@ -1,6 +0,0 @@ - -Starting up - -- npm install -- gulp -- run mitmweb and open http://localhost:8081/ diff --git a/web/README.md b/web/README.md new file mode 100644 index 00000000..c43d09f0 --- /dev/null +++ b/web/README.md @@ -0,0 +1,6 @@ +# Quick Start + + +- Run `yarn` to install dependencies +- Run `gulp` to start live-compilation. +- Run `mitmweb` and open http://localhost:8081/ diff --git a/web/src/js/components/Header/FilterDocs.jsx b/web/src/js/components/Header/FilterDocs.jsx index b471840b..198c0117 100644 --- a/web/src/js/components/Header/FilterDocs.jsx +++ b/web/src/js/components/Header/FilterDocs.jsx @@ -44,7 +44,7 @@ export default class FilterDocs extends Component { ))} <tr key="docs-link"> <td colSpan="2"> - <a href="http://docs.mitmproxy.org/en/stable/features/filters.html" + <a href="https://mitmproxy.org/docs/latest/concepts-filters/" target="_blank"> <i className="fa fa-external-link"></i> mitmproxy docs</a> diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index e4b151ad..80f43018 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -253,7 +253,7 @@ BindingExpr / Expr /* All the filters except "~s" and "~src" are arranged in the ascending order as - given in the docs(http://docs.mitmproxy.org/en/latest/features/filters.html). + given in the docs(https://mitmproxy.org/docs/latest/concepts-filters/). "~s" and "~src" are so arranged as "~s" caused problems in the evaluation of "~src". */ |
