diff options
34 files changed, 583 insertions, 453 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 6891f1b3..90364823 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,22 +9,8 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python35" - TOXENV: "py35" - # TODO: ENABLE WHEN AVAILABLE - # - PYTHON: "C:\\Python36" - # TOXENV: "py36" - - SNAPSHOT_HOST: - secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= - SNAPSHOT_PORT: - secure: TiJPtg60/edYTH8RnoBErg== - SNAPSHOT_USER: - secure: 6yBwmO5gv4vAwoFYII8qjQ== - SNAPSHOT_PASS: - secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV - RTOOL_KEY: - secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM + - PYTHON: "C:\\Python36" + TOXENV: "py36" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -36,17 +22,10 @@ test_script: - ps: | $Env:VERSION = $(python -m mitmproxy.version) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" - tox -e rtool -- wheel - tox -e rtool -- bdist - + tox -e cibuild -- build - ps: | - if( - ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - echo "Decrypt license..." - tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml - $ibVersion = "17.12.0" + if($Env:RTOOL_KEY) { + $ibVersion = "18.2.0" $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe" $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe" if (!(Test-Path $ibSetup)) { @@ -58,8 +37,8 @@ test_script: } echo "Install InstallBuilder..." Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait - # Wait until executable exists - no idea why this is necessary. - while (!(Test-Path $ibCli)) { Start-Sleep 0.1 } + echo "Decrypt license..." + tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml echo "Run InstallBuilder..." &$ibCli ` build ` @@ -73,15 +52,7 @@ test_script: } deploy_script: - # we build binaries on every run, but we only upload them for master snapshots or tags. - ps: | - if( - ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts - tox -e rtool -- upload-snapshot --bdist --wheel --installer - } + ps: tox -e cibuild -- upload cache: - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml diff --git a/.travis.yml b/.travis.yml index c22038fd..66a4f61e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,12 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 BDIST=1 + env: TOXENV=py36 BDIST=1 - python: 3.5 - env: TOXENV=py35 BDIST=1 + env: TOXENV=py35 dist: precise - python: 3.6 - env: TOXENV=py36 + env: TOXENV=py36 BDIST=1 WHEEL=1 - python: 3.6 env: TOXENV=individual_coverage - language: node_js @@ -42,6 +42,17 @@ matrix: yarn: true directories: - web/node_modules + - language: python + env: + - NAME=docs + install: + - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + - sudo dpkg -i hugo*.deb + - pip install awscli + script: + - cd docs && ./ci + after_success: + - echo done install: - | @@ -50,12 +61,11 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3 - pyenv global 3.5.3 - pyenv shell 3.5.3 - pip install -U pip setuptools wheel virtualenv + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4 + pyenv global 3.6.4 + pyenv shell 3.6.4 fi - - pip install tox + - pip install tox virtualenv setuptools script: - tox -- --verbose --cov-report=term @@ -63,16 +73,14 @@ script: if [[ $BDIST == "1" ]] then git fetch --unshallow --tags - tox -e rtool -- bdist + tox -e cibuild -- build fi after_success: - # we build binaries on every run, but we only upload them for master snapshots or tags. - | - if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" ]] then - tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts - tox -e rtool -- upload-snapshot --bdist + tox -e cibuild -- upload fi notifications: @@ -45,7 +45,7 @@ Join our developer chat on Slack if you would like to contribute to mitmproxy it Installation ------------ -The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`__. +The installation instructions are `here <https://mitmproxy.org/docs/latest/overview-installation>`__. If you want to contribute changes, keep on reading. Contributing @@ -182,7 +182,7 @@ with the following command: :target: https://pypi.python.org/pypi/mitmproxy :alt: Supported Python versions -.. _`advanced installation`: http://docs.mitmproxy.org/en/latest/install.html#advanced-installation +.. _`advanced installation`: https://mitmproxy.org/docs/latest/overview-installation/#advanced-installation .. _virtualenv: https://virtualenv.pypa.io/ .. _`pytest`: http://pytest.org/ .. _tox: https://tox.readthedocs.io/ diff --git a/docs/bucketassets/error.html b/docs/bucketassets/error.html new file mode 100644 index 00000000..a855e72b --- /dev/null +++ b/docs/bucketassets/error.html @@ -0,0 +1,9 @@ +Not found +<html> + <head> + <meta http-equiv="refresh" content="0;URL='/stable'" /> + </head> + <body> + Not found - redirecting you to <a href="/stable">latest stable docs</a>. + </body> +</html> diff --git a/docs/bucketassets/robots.txt b/docs/bucketassets/robots.txt new file mode 100644 index 00000000..da4c47d6 --- /dev/null +++ b/docs/bucketassets/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /archive/ +Disallow: /master/
\ No newline at end of file diff --git a/docs/build b/docs/build new file mode 100755 index 00000000..1ca3fdb8 --- /dev/null +++ b/docs/build @@ -0,0 +1,3 @@ +#!/bin/sh + +cd src; hugo
\ No newline at end of file diff --git a/docs/ci b/docs/ci new file mode 100755 index 00000000..1584c5e1 --- /dev/null +++ b/docs/ci @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script gets run from CI to render and upload docs + +./build + +# Only upload if we have defined credentials - we only have these defined for +# trusted commits (i.e. not PRs). +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then + aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master + aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/master" +fi diff --git a/docs/setup b/docs/setup new file mode 100755 index 00000000..8a9c31fd --- /dev/null +++ b/docs/setup @@ -0,0 +1,7 @@ +#!/bin/sh + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/error.html s3://docs.mitmproxy.org/error.html +aws --profile mitmproxy \ + s3 cp --acl public-read ./bucketassets/robots.txt s3://docs.mitmproxy.org/robots.txt diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md new file mode 100644 index 00000000..fc056545 --- /dev/null +++ b/docs/src/content/concepts-protocols.md @@ -0,0 +1,93 @@ +--- +title: "Protocols" +menu: + concepts: + weight: 7 +--- + +# Protocols + +## HTTP/1.0 and HTTP/1.1 + +[RFC7230: HTTP/1.1: Message Syntax and Routing](http://tools.ietf.org/html/rfc7230) + +[RFC7231: HTTP/1.1: Semantics and Content](http://tools.ietf.org/html/rfc7231) + +HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, +which takes care of all semantics and on-the-wire parsing/serialization tasks. + +mitmproxy currently does not support HTTP trailers - but if you want to send +us a PR, we promise to take look! + +## HTTP/2 + +[RFC7540: Hypertext Transfer Protocol Version 2 (HTTP/2)](http://tools.ietf.org/html/rfc7540>) + +HTTP/2 support in mitmproxy is based on +[hyper-h2](https://github.com/python-hyper/hyper-h2). It fully encapsulates the +internal state of HTTP/2 connections and provides an easy-to-use event-based +API. mitmproxy supports the majority of HTTP/2 feature and tries to +transparently pass-through as much information as possible. + +mitmproxy currently does not support HTTP/2 trailers - but if you want to send +us a PR, we promise to take look! + +mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the +major browser vendors have implemented it. + +Some websites are still having problems with correct HTTP/2 support in their +webservers and can cause errors, dropped connectiones, or simply no response at +all. We are trying to be as tolerant and forgiving as possible with the types of +data we send and receive, but +[some](https://github.com/mitmproxy/mitmproxy/issues/1745) +[faulty](https://github.com/mitmproxy/mitmproxy/issues/2823) +[implementations](https://github.com/mitmproxy/mitmproxy/issues/1824) +[simply](https://github.com/mitmproxy/mitmproxy/issues/1891) don't work well +with mitmproxy. + +In order to increase the compatibility of mitmproxy with HTTP/2 webservers, we +default to NOT forward any priority information that is sent by a client. You +can enable it with: `http2_priority=true`. + +## WebSocket + +[RFC6455: The WebSocket Protocol](http://tools.ietf.org/html/rfc6455) + +[RFC7692: Compression Extensions for WebSocket](http://tools.ietf.org/html/rfc7692) + +WebSocket support in mitmproxy is based on [wsproto] +(https://github.com/python-hyper/wsproto) project. It fully encapsulates +WebSocket frames/messages/connections and provides an easy-to-use event-based +API. + +mitmproxy fully supports the compression extension for WebSocket messages, +provided by wsproto. Message contents are automatically compressed and +decompressed before firing events. + +mitmproxy currently does not display WebSocket messages in the console or web +UI. Only the WebSocket handshake flow is shown, which contains a reference to +the parent flow for all messages exchanged over this connection. + +If an endpoint sends a PING to mitmproxy, a PONG will be sent back immediately +(with the same payload if present). To keep the other connection alive, a new +PING (without a payload) is sent to the other endpoint. Unsolicited PONG's are +not forwarded. All PING's and PONG's are logged (with payload if present). + +## Raw TCP / TCP Proxy / Fallback + +In case mitmproxy does not handle a specific protocol, you can exempt +hostnames from processing, so that mitmproxy acts as a generic TCP forwarder. +This feature is closely related to the *passthrough* functionality, +but differs in two important aspects: + + * The raw TCP messages are printed to the event log. + * SSL connections will be intercepted. + +Please note that message interception or modification are not possible yet. If +you are not interested in the raw TCP messages, you should use the ignore +domains feature. + +| | | +| ------------------ | ------------------ | +| command-line alias | `--tcp HOST` | +| mitmproxy shortcut | press `O` then `T` | diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md new file mode 100644 index 00000000..588223ac --- /dev/null +++ b/docs/src/content/howto-wireshark-tls.md @@ -0,0 +1,28 @@ +--- +title: "Wireshark and SSL/TLS" +menu: + howto: + weight: 1 +--- + +# Wireshark and SSL/TLS Master Secrets + +The SSL/SSL master keys can be logged by mitmproxy so that external programs can +decrypt SSL/TLS connections both from and to the proxy. Recent versions of +Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. + +Key logging is enabled by setting the environment variable `SSLKEYLOGFILE` so +that it points to a writable text file: +{{< highlight bash >}} +SSLKEYLOGFILE="$PWD/.mitmproxy/sslkeylogfile.txt" mitmproxy +{{< / highlight >}} +You can also `export` this environment variable to make it persistent for all applications started from your current shell session. + +You can specify the key file path in Wireshark via `Edit -> Preferences -> +Protocols -> SSL -> (Pre)-Master-Secret log filename`. If your SSLKEYLOGFILE +does not exist yet, just create an empty text file, so you can select it in +Wireshark (or run mitmproxy to create and collect master secrets). + +Note that `SSLKEYLOGFILE` is respected by other programs as well, e.g., Firefox +and Chrome. If this creates any issues, you can use `MITMPROXY_SSLKEYLOGFILE` +instead without affecting other applications. diff --git a/docs/src/themes/mitmproxydocs/static/css/style.css b/docs/src/themes/mitmproxydocs/static/css/style.css index 14823447..868c7d0a 100644 --- a/docs/src/themes/mitmproxydocs/static/css/style.css +++ b/docs/src/themes/mitmproxydocs/static/css/style.css @@ -6718,7 +6718,6 @@ label.panel-block { padding: 3rem 1.5rem 6rem; } .sidebody { - height: 100vh; overflow-x: hidden; overflow-y: scroll; } @@ -6731,17 +6730,17 @@ label.panel-block { width: 100%; text-align: right; } -.sidebar .version { - padding: 1em; } - -.sidebar .brand { - background-color: #303030; - color: #c0c0c0; - padding: 1em; - top: 0; } - -.sidebar .menu { - padding: 1em; } +.sidebar { + background-color: #F1F1F1; } + .sidebar .version { + padding: 1em; } + .sidebar .brand { + background-color: #303030; + color: #c0c0c0; + padding: 1em; + top: 0; } + .sidebar .menu { + padding: 1em; } .mainbody { padding: 3em; } diff --git a/docs/style/style.scss b/docs/style/style.scss index 2c66a4c9..bc146fd5 100644 --- a/docs/style/style.scss +++ b/docs/style/style.scss @@ -11,7 +11,6 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox @import "../node_modules/bulma/sass/layout/_all"; .sidebody { - height: 100vh; overflow-x: hidden; overflow-y: scroll; } @@ -28,8 +27,8 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox margin-bottom: 1em; } - .sidebar { + background-color: #F1F1F1; .version { padding: 1em; } diff --git a/docs/upload-archive b/docs/upload-archive new file mode 100755 index 00000000..86dd248e --- /dev/null +++ b/docs/upload-archive @@ -0,0 +1,17 @@ +#!/bin/sh + +if [[ $# -eq 0 ]] ; then + echo "Please supply a version, e.g. 'v3'" + exit 1 +fi + +# This script uploads docs to a specified archive version. + +SPATH="/archive/$1" + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "$SPATH" diff --git a/docs/upload-stable b/docs/upload-stable new file mode 100755 index 00000000..c2c1267e --- /dev/null +++ b/docs/upload-stable @@ -0,0 +1,8 @@ +#!/bin/sh + +aws configure set preview.cloudfront true +aws --profile mitmproxy \ + s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/stable" diff --git a/examples/addons/events.py b/examples/addons/events.py index d3c90430..674ab4c8 100644 --- a/examples/addons/events.py +++ b/examples/addons/events.py @@ -80,12 +80,12 @@ class Events: attribute. """ - def websocket_start(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_start(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has commenced. """ - def websocket_message(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_message(self, flow: mitmproxy.websocket.WebSocketFlow): """ Called when a WebSocket message is received from the client or server. The most recent message will be flow.messages[-1]. The @@ -93,12 +93,12 @@ class Events: messages, corresponding to the BINARY and TEXT frame types. """ - def websocket_error(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_error(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has had an error. """ - def websocket_end(self, flow: mitmproxy.websocket.WebsocketFlow): + def websocket_end(self, flow: mitmproxy.websocket.WebSocketFlow): """ A websocket connection has ended. """ diff --git a/mitmproxy/addons/browser.py b/mitmproxy/addons/browser.py index 247c356b..2f8eeccd 100644 --- a/mitmproxy/addons/browser.py +++ b/mitmproxy/addons/browser.py @@ -15,6 +15,7 @@ def get_chrome_executable() -> typing.Optional[str]: r"C:\Program Files (x86)\Google\Application\chrome.exe", # Linux binary names from Python's webbrowser module. "google-chrome", + "google-chrome-stable", "chrome", "chromium", "chromium-browser", diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 6b4041f6..959fdd33 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -429,10 +429,7 @@ class Request(message.Message): def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: - try: - return tuple(mitmproxy.net.http.url.decode(self.content.decode())) - except ValueError: - pass + return tuple(mitmproxy.net.http.url.decode(self.get_text(strict=False))) return () def _set_urlencoded_form(self, form_data): @@ -441,7 +438,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() + self.content = mitmproxy.net.http.url.encode(form_data, self.get_text(strict=False)).encode() @property def urlencoded_form(self): diff --git a/mitmproxy/platform/osx.py b/mitmproxy/platform/osx.py index f9de1fbf..9b150679 100644 --- a/mitmproxy/platform/osx.py +++ b/mitmproxy/platform/osx.py @@ -32,5 +32,5 @@ def original_addr(csock): if insufficient_priv: raise RuntimeError( "Insufficient privileges to access pfctl. " - "See http://docs.mitmproxy.org/en/latest/transparent/osx.html for details.") + "See https://mitmproxy.org/docs/latest/howto-transparent/#macos for details.") return pf.lookup(peer[0], peer[1], stxt) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 439beb3d..410ab701 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -53,7 +53,7 @@ class ProxyConfig: if not os.path.exists(os.path.dirname(certstore_path)): raise exceptions.OptionsError( "Certificate Authority parent directory does not exist: %s" % - os.path.dirname(options.cadir) + os.path.dirname(certstore_path) ) self.certstore = certs.CertStore.from_store( certstore_path, diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 9420c105..650a9366 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -30,12 +30,14 @@ class FlowViewHeader(urwid.WidgetWrap): self.focus_changed() def focus_changed(self): + cols, _ = self.master.ui.get_cols_rows() if self.master.view.focus.flow: self._w = common.format_flow( self.master.view.focus.flow, False, extended=True, - hostheader=self.master.options.showhost + hostheader=self.master.options.showhost, + max_url_len=cols, ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 5da6ef0b..5cc1cf43 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -94,7 +94,9 @@ class ConsoleMaster(master.Master): self.start_err = entry else: signals.status_message.send( - message=(entry.level, "{}: {}".format(entry.level.title(), entry.msg)), + message=(entry.level, + "{}: {}".format(entry.level.title(), + str(entry.msg).lstrip())), expire=5 ) diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 8553a66f..d601968e 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -49,7 +49,8 @@ class ActionBar(urwid.WidgetWrap): def sig_message(self, sender, message, expire=1): if self.prompting: return - w = urwid.Text(message) + cols, _ = self.master.ui.get_cols_rows() + w = urwid.Text(self.shorten_message(message, cols)) self._w = w if expire: def cb(*args): @@ -60,6 +61,36 @@ class ActionBar(urwid.WidgetWrap): def prep_prompt(self, p): return p.strip() + ": " + def shorten_message(self, msg, max_width): + """ + Shorten message so that it fits into a single line in the statusbar. + """ + if isinstance(msg, tuple): + disp_attr, msg_text = msg + elif isinstance(msg, str): + disp_attr, msg_text = None, msg + else: + return msg + msg_end = "\u2026" # unicode ellipsis for the end of shortened message + prompt = "(more in eventlog)" + + msg_lines = msg_text.split("\n") + first_line = msg_lines[0] + if len(msg_lines) > 1: + # First line of messages with a few lines must end with prompt. + line_length = len(first_line) + len(prompt) + else: + line_length = len(first_line) + + if line_length > max_width: + shortening_index = max(0, max_width - len(prompt) - len(msg_end)) + first_line = first_line[:shortening_index] + msg_end + else: + if len(msg_lines) == 1: + prompt = "" + + return [(disp_attr, first_line), ("warn", prompt)] + def sig_prompt(self, sender, prompt, text, callback, args=()): signals.focus.send(self, section="footer") self._w = urwid.Edit(self.prep_prompt(prompt), text or "") diff --git a/mitmproxy/tools/web/static/app.js b/mitmproxy/tools/web/static/app.js index ace3106d..ac3be38c 100644 --- a/mitmproxy/tools/web/static/app.js +++ b/mitmproxy/tools/web/static/app.js @@ -86,7 +86,7 @@ "use strict";function _interopRequireWildcard(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&(t[r]=e[r]);return t.default=e,t}function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function FileMenu(e){var t=e.clearFlows,r=e.loadFlows,a=e.saveFlows,l=e.openModal;return _react2.default.createElement(_Dropdown2.default,{className:"pull-left",btnClass:"special",text:"mitmproxy"},_react2.default.createElement("a",{href:"#",onClick:function(e){return FileMenu.onNewClick(e,t)}},_react2.default.createElement("i",{className:"fa fa-fw fa-file"})," New"),_react2.default.createElement(_FileChooser2.default,{icon:"fa-folder-open",text:" Open...",onOpenFile:function(e){return r(e)}}),_react2.default.createElement("a",{href:"#",onClick:function(e){e.preventDefault(),a()}},_react2.default.createElement("i",{className:"fa fa-fw fa-floppy-o"})," Save..."),_react2.default.createElement(_HideInStatic2.default,null,_react2.default.createElement("a",{href:"#",onClick:function(e){e.preventDefault(),l()}},_react2.default.createElement("i",{className:"fa fa-fw fa-cog"})," Options"),_react2.default.createElement(_Dropdown.Divider,null),_react2.default.createElement("a",{href:"http://mitm.it/",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-fw fa-external-link"})," Install Certificates...")))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.FileMenu=FileMenu;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_reactRedux=require("react-redux"),_FileChooser=require("../common/FileChooser"),_FileChooser2=_interopRequireDefault(_FileChooser),_Dropdown=require("../common/Dropdown"),_Dropdown2=_interopRequireDefault(_Dropdown),_flows=require("../../ducks/flows"),flowsActions=_interopRequireWildcard(_flows),_modal=require("../../ducks/ui/modal"),modalActions=_interopRequireWildcard(_modal),_HideInStatic=require("../common/HideInStatic"),_HideInStatic2=_interopRequireDefault(_HideInStatic);FileMenu.propTypes={clearFlows:_propTypes2.default.func.isRequired,loadFlows:_propTypes2.default.func.isRequired,saveFlows:_propTypes2.default.func.isRequired,openModal:_propTypes2.default.func.isRequired},FileMenu.onNewClick=function(e,t){e.preventDefault(),confirm("Delete all flows?")&&t()},exports.default=(0,_reactRedux.connect)(null,{clearFlows:flowsActions.clear,loadFlows:flowsActions.upload,saveFlows:flowsActions.download,openModal:function(){return modalActions.setActiveModal("OptionModal")}})(FileMenu); },{"../../ducks/flows":57,"../../ducks/ui/modal":65,"../common/Dropdown":48,"../common/FileChooser":49,"../common/HideInStatic":50,"prop-types":"prop-types","react":"react","react-redux":"react-redux"}],30:[function(require,module,exports){ -"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_utils=require("../../utils"),FilterDocs=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={doc:t.doc},n}return _inherits(t,e),_createClass(t,[{key:"componentWillMount",value:function(){var e=this;t.xhr||(t.xhr=(0,_utils.fetchApi)("/filter-help").then(function(e){return e.json()}),t.xhr.catch(function(){t.xhr=null})),this.state.doc||t.xhr.then(function(r){t.doc=r,e.setState({doc:r})})}},{key:"render",value:function(){var e=this,t=this.state.doc;return t?_react2.default.createElement("table",{className:"table table-condensed"},_react2.default.createElement("tbody",null,t.commands.map(function(t){return _react2.default.createElement("tr",{key:t[1],onClick:function(r){return e.props.selectHandler(t[0].split(" ")[0]+" ")}},_react2.default.createElement("td",null,t[0].replace(" "," ")),_react2.default.createElement("td",null,t[1]))}),_react2.default.createElement("tr",{key:"docs-link"},_react2.default.createElement("td",{colSpan:"2"},_react2.default.createElement("a",{href:"http://docs.mitmproxy.org/en/stable/features/filters.html",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-external-link"})," mitmproxy docs"))))):_react2.default.createElement("i",{className:"fa fa-spinner fa-spin"})}}]),t}(_react.Component);FilterDocs.xhr=null,FilterDocs.doc=null,exports.default=FilterDocs; +"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_utils=require("../../utils"),FilterDocs=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={doc:t.doc},n}return _inherits(t,e),_createClass(t,[{key:"componentWillMount",value:function(){var e=this;t.xhr||(t.xhr=(0,_utils.fetchApi)("/filter-help").then(function(e){return e.json()}),t.xhr.catch(function(){t.xhr=null})),this.state.doc||t.xhr.then(function(r){t.doc=r,e.setState({doc:r})})}},{key:"render",value:function(){var e=this,t=this.state.doc;return t?_react2.default.createElement("table",{className:"table table-condensed"},_react2.default.createElement("tbody",null,t.commands.map(function(t){return _react2.default.createElement("tr",{key:t[1],onClick:function(r){return e.props.selectHandler(t[0].split(" ")[0]+" ")}},_react2.default.createElement("td",null,t[0].replace(" "," ")),_react2.default.createElement("td",null,t[1]))}),_react2.default.createElement("tr",{key:"docs-link"},_react2.default.createElement("td",{colSpan:"2"},_react2.default.createElement("a",{href:"https://mitmproxy.org/docs/latest/concepts-filters/",target:"_blank"},_react2.default.createElement("i",{className:"fa fa-external-link"})," mitmproxy docs"))))):_react2.default.createElement("i",{className:"fa fa-spinner fa-spin"})}}]),t}(_react.Component);FilterDocs.xhr=null,FilterDocs.doc=null,exports.default=FilterDocs; },{"../../utils":71,"react":"react"}],31:[function(require,module,exports){ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var o=t[r];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(e,o.key,o)}}return function(t,r,o){return r&&e(t.prototype,r),o&&e(t,o),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_reactDom=require("react-dom"),_reactDom2=_interopRequireDefault(_reactDom),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames),_utils=require("../../utils.js"),_filt=require("../../filt/filt"),_filt2=_interopRequireDefault(_filt),_FilterDocs=require("./FilterDocs"),_FilterDocs2=_interopRequireDefault(_FilterDocs),FilterInput=function(e){function t(e,r){_classCallCheck(this,t);var o=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return o.state={value:o.props.value,focus:!1,mousefocus:!1},o.onChange=o.onChange.bind(o),o.onFocus=o.onFocus.bind(o),o.onBlur=o.onBlur.bind(o),o.onKeyDown=o.onKeyDown.bind(o),o.onMouseEnter=o.onMouseEnter.bind(o),o.onMouseLeave=o.onMouseLeave.bind(o),o.selectFilter=o.selectFilter.bind(o),o}return _inherits(t,e),_createClass(t,[{key:"componentWillReceiveProps",value:function(e){this.setState({value:e.value})}},{key:"isValid",value:function(e){try{var t=null==e?this.state.value:e;return t&&_filt2.default.parse(t),!0}catch(e){return!1}}},{key:"getDesc",value:function(){if(!this.state.value)return _react2.default.createElement(_FilterDocs2.default,{selectHandler:this.selectFilter});try{return _filt2.default.parse(this.state.value).desc}catch(e){return""+e}}},{key:"onChange",value:function(e){var t=e.target.value;this.setState({value:t}),this.isValid(t)&&this.props.onChange(t)}},{key:"onFocus",value:function(){this.setState({focus:!0})}},{key:"onBlur",value:function(){this.setState({focus:!1})}},{key:"onMouseEnter",value:function(){this.setState({mousefocus:!0})}},{key:"onMouseLeave",value:function(){this.setState({mousefocus:!1})}},{key:"onKeyDown",value:function(e){e.keyCode!==_utils.Key.ESC&&e.keyCode!==_utils.Key.ENTER||(this.blur(),this.setState({mousefocus:!1})),e.stopPropagation()}},{key:"selectFilter",value:function(e){this.setState({value:e}),_reactDom2.default.findDOMNode(this.refs.input).focus()}},{key:"blur",value:function(){_reactDom2.default.findDOMNode(this.refs.input).blur()}},{key:"select",value:function(){_reactDom2.default.findDOMNode(this.refs.input).select()}},{key:"render",value:function(){var e=this.props,t=e.type,r=e.color,o=e.placeholder,n=this.state,a=n.value,s=n.focus,u=n.mousefocus;return _react2.default.createElement("div",{className:(0,_classnames2.default)("filter-input input-group",{"has-error":!this.isValid()})},_react2.default.createElement("span",{className:"input-group-addon"},_react2.default.createElement("i",{className:"fa fa-fw fa-"+t,style:{color:r}})),_react2.default.createElement("input",{type:"text",ref:"input",placeholder:o,className:"form-control",value:a,onChange:this.onChange,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown}),(s||u)&&_react2.default.createElement("div",{className:"popover bottom",onMouseEnter:this.onMouseEnter,onMouseLeave:this.onMouseLeave},_react2.default.createElement("div",{className:"arrow"}),_react2.default.createElement("div",{className:"popover-content"},this.getDesc())))}}]),t}(_react.Component);exports.default=FilterInput; @@ -137,7 +137,7 @@ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function Button(e){var t=e.onClick,r=e.children,a=e.icon,s=e.disabled,l=e.className,i=e.title;return _react2.default.createElement("div",{className:(0,_classnames2.default)(l,"btn btn-default"),onClick:!s&&t,disabled:s,title:i},a&&_react2.default.createElement("i",{className:"fa fa-fw "+a}),r)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=Button;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames);Button.propTypes={onClick:_propTypes2.default.func.isRequired,children:_propTypes2.default.node.isRequired,icon:_propTypes2.default.string,title:_propTypes2.default.string}; },{"classnames":"classnames","prop-types":"prop-types","react":"react"}],47:[function(require,module,exports){ -"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function DocsLink(e){var r=e.children,t=e.resource,a="http://docs.mitmproxy.org/en/stable/"+t;return _react2.default.createElement("a",{target:"_blank",href:a},r||_react2.default.createElement("i",{className:"fa fa-question-circle"}))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=DocsLink;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes);DocsLink.propTypes={resource:_propTypes2.default.string.isRequired}; +"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function DocsLink(e){var r=e.children,t=e.resource,a="https://mitmproxy.org/docs/latest"+t;return _react2.default.createElement("a",{target:"_blank",href:a},r||_react2.default.createElement("i",{className:"fa fa-question-circle"}))}Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=DocsLink;var _react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes);DocsLink.propTypes={resource:_propTypes2.default.string.isRequired}; },{"prop-types":"prop-types","react":"react"}],48:[function(require,module,exports){ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0}),exports.Divider=void 0;var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_react=require("react"),_react2=_interopRequireDefault(_react),_propTypes=require("prop-types"),_propTypes2=_interopRequireDefault(_propTypes),_classnames=require("classnames"),_classnames2=_interopRequireDefault(_classnames),Divider=exports.Divider=function(){return _react2.default.createElement("hr",{className:"divider"})},Dropdown=function(e){function t(e,r){_classCallCheck(this,t);var n=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e,r));return n.state={open:!1},n.close=n.close.bind(n),n.open=n.open.bind(n),n}return _inherits(t,e),_createClass(t,[{key:"close",value:function(){this.setState({open:!1}),document.removeEventListener("click",this.close)}},{key:"open",value:function(e){e.preventDefault(),this.state.open||(this.setState({open:!this.state.open}),document.addEventListener("click",this.close))}},{key:"render",value:function(){var e=this.props,t=e.dropup,r=e.className,n=e.btnClass,o=e.text,s=e.children;return _react2.default.createElement("div",{className:(0,_classnames2.default)(t?"dropup":"dropdown",r,{open:this.state.open})},_react2.default.createElement("a",{href:"#",className:n,onClick:this.open},o),_react2.default.createElement("ul",{className:"dropdown-menu",role:"menu"},s.map(function(e,t){return _react2.default.createElement("li",{key:t}," ",e," ")})))}}]),t}(_react.Component);Dropdown.propTypes={dropup:_propTypes2.default.bool,className:_propTypes2.default.string,btnClass:_propTypes2.default.string.isRequired},Dropdown.defaultProps={dropup:!1},exports.default=Dropdown; diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 66257852..253503f4 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -57,7 +57,7 @@ class WebSocketMessage(serializable.Serializable): class WebSocketFlow(flow.Flow): """ - A WebsocketFlow is a simplified representation of a Websocket connection. + A WebSocketFlow is a simplified representation of a Websocket connection. """ def __init__(self, client_conn, server_conn, handshake_flow, live=None): diff --git a/release/ci.py b/release/ci.py new file mode 100755 index 00000000..94b1f13d --- /dev/null +++ b/release/ci.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 + +import contextlib +import os +import platform +import sys +import shutil +import subprocess +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename + +import cryptography.fernet +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] +else: + VENV_BIN = "bin" + PYINSTALLER_ARGS = [] + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") + +BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +# PyInstaller 3.2 does not bundle pydivert's Windivert binaries +PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + +VENV_DIR = join(BUILD_DIR, "venv") + +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] + +TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) +BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) +if TAG: + VERSION = TAG + UPLOAD_DIR = VERSION +elif BRANCH: + VERSION = BRANCH + UPLOAD_DIR = "branches/%s" % VERSION +else: + print("Could not establish build name - exiting." % BRANCH) + sys.exit(0) + + +print("BUILD VERSION=%s" % VERSION) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=VERSION, + platform=PLATFORM_TAG, + ext=ext + ) + + +def wheel_name() -> str: + return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) + + +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=VERSION, + platform=PLATFORM_TAG, + ext=ext, + ) + + +@contextlib.contextmanager +def chdir(path: str): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("info") +def info(): + print("Version: %s" % VERSION) + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + os.makedirs(DIST_DIR, exist_ok=True) + + if "WHEEL" in os.environ: + print("Building wheel...") + subprocess.check_call( + [ + "python", + "setup.py", "-q", "bdist_wheel", + "--dist-dir", "release/dist", + ] + ) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + print("> %s --version" % executable) + print(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) + + +def is_pr(): + if "TRAVIS_PULL_REQUEST" in os.environ: + if os.environ["TRAVIS_PULL_REQUEST"] == "false": + return False + return True + elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"): + return True + return False + + +@cli.command("upload") +def upload(): + """ + Upload snapshot to snapshot server + """ + # This requires some explanation. The AWS access keys are only exposed to + # privileged builds - that is, they are not available to PRs from forks. + # However, they ARE exposed to PRs from a branch within the main repo. This + # check catches that corner case, and prevents an inadvertent upload. + if is_pr(): + print("Refusing to upload a pull request") + return + if "AWS_ACCESS_KEY_ID" in os.environ: + subprocess.check_call( + [ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, + "--recursive", + ] + ) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + +if __name__ == "__main__": + cli() diff --git a/release/known_hosts.enc b/release/known_hosts.enc deleted file mode 100644 index 585ee678..00000000 --- a/release/known_hosts.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7
\ No newline at end of file diff --git a/release/rtool.pem b/release/rtool.pem deleted file mode 100644 index 097dff1a..00000000 --- a/release/rtool.pem +++ /dev/null @@ -1,68 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB - -0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y -byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU -V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z -T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh -PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl -w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD -P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt -JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE -yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km -cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe -hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8 -jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx -QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey -OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL -kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR -AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc -sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko -SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz -5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P -cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk -kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/ -Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9 -3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4 -mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t -8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j -FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf -EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP -a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR -AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+ -m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P -bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q -QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx -uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq -yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K -u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK -MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0 -8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq -y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz -G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi -o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq -2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL -5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp -30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac -7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA -1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8 -eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB -bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO -VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+ -2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175 ------END RSA PRIVATE KEY----- ------BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y -qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe -24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX -FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ -0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI -KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt -0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+ -tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ -TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ -q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J -6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD -5I1jQDXFbjAgAnaSyqyW368CAwEAAQ== ------END PUBLIC KEY-----
\ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 25f37e61..0009cf89 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,79 +1,22 @@ #!/usr/bin/env python3 import contextlib -import fnmatch import os import sys import platform -import re import runpy import shlex -import shutil import subprocess -import tarfile -import zipfile -from os.path import join, abspath, dirname, exists, basename +from os.path import join, abspath, dirname -import click import cryptography.fernet -import pysftp - -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if platform.system() == "Windows": - VENV_BIN = "Scripts" - PYINSTALLER_ARGS = [ - # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. - "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", - ] -else: - VENV_BIN = "bin" - PYINSTALLER_ARGS = [] - -# ZipFile and tarfile have slightly different APIs. Fix that. -if platform.system() == "Windows": - def Archive(name): - a = zipfile.ZipFile(name, "w") - a.add = a.write - return a -else: - def Archive(name): - return tarfile.open(name, "w:gz") +import click -PLATFORM_TAG = { - "Darwin": "osx", - "Windows": "windows", - "Linux": "linux", -}.get(platform.system(), platform.system()) ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") - -BUILD_DIR = join(RELEASE_DIR, "build") DIST_DIR = join(RELEASE_DIR, "dist") - -PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") -# PyInstaller 3.2 does not bundle pydivert's Windivert binaries -PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") -PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") -PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) - -VENV_DIR = join(BUILD_DIR, "venv") - -# Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") -BDISTS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] -} -if platform.system() == "Windows": - BDISTS["mitmproxy"].remove("mitmproxy") - -TOOLS = [ - tool - for tools in sorted(BDISTS.values()) - for tool in tools -] def git(args: str) -> str: @@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str: return x["get_version"](dev, build, True) -def set_version(dev: bool) -> None: - """ - Update version information in mitmproxy's version.py to either include hardcoded information or not. - """ - version = get_version(dev, dev) - with open(VERSION_FILE, "r") as f: - content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) - with open(VERSION_FILE, "w") as f: - f.write(content) - - -def archive_name(bdist: str) -> str: - if platform.system() == "Windows": - ext = "zip" - else: - ext = "tar.gz" - return "{project}-{version}-{platform}.{ext}".format( - project=bdist, - version=get_version(), - platform=PLATFORM_TAG, - ext=ext - ) - - def wheel_name() -> str: return "mitmproxy-{version}-py3-none-any.whl".format( version=get_version(True), ) -def installer_name() -> str: - ext = { - "Windows": "exe", - "Darwin": "dmg", - "Linux": "run" - }[platform.system()] - return "mitmproxy-{version}-{platform}-installer.{ext}".format( - version=get_version(), - platform=PLATFORM_TAG, - ext=ext, - ) - - @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -146,24 +51,6 @@ def cli(): pass -@cli.command("encrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def encrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.encrypt(infile.read())) - - -@cli.command("decrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.decrypt(infile.read())) - - @cli.command("contributors") def contributors(): """ @@ -176,97 +63,6 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("wheel") -def make_wheel(): - """ - Build a Python wheel - """ - set_version(True) - try: - subprocess.check_call([ - "tox", "-e", "wheel", - ], env={ - **os.environ, - "VERSION": get_version(True), - }) - finally: - set_version(False) - - -@cli.command("bdist") -def make_bdist(): - """ - Build a binary distribution - """ - if exists(PYINSTALLER_TEMP): - shutil.rmtree(PYINSTALLER_TEMP) - if exists(PYINSTALLER_DIST): - shutil.rmtree(PYINSTALLER_DIST) - - os.makedirs(DIST_DIR, exist_ok=True) - - for bdist, tools in sorted(BDISTS.items()): - with Archive(join(DIST_DIR, archive_name(bdist))) as archive: - for tool in tools: - # We can't have a folder and a file with the same name. - if tool == "mitmproxy": - tool = "mitmproxy_main" - # This is PyInstaller, so it messes up paths. - # We need to make sure that we are in the spec folder. - with chdir(PYINSTALLER_SPEC): - print("Building %s binary..." % tool) - excludes = [] - if tool != "mitmweb": - excludes.append("mitmproxy.tools.web") - if tool != "mitmproxy_main": - excludes.append("mitmproxy.tools.console") - - # Overwrite mitmproxy/version.py to include commit info - set_version(True) - try: - subprocess.check_call( - [ - "pyinstaller", - "--clean", - "--workpath", PYINSTALLER_TEMP, - "--distpath", PYINSTALLER_DIST, - "--additional-hooks-dir", PYINSTALLER_HOOKS, - "--onefile", - "--console", - "--icon", "icon.ico", - # This is PyInstaller, so setting a - # different log level obviously breaks it :-) - # "--log-level", "WARN", - ] - + [x for e in excludes for x in ["--exclude-module", e]] - + PYINSTALLER_ARGS - + [tool] - ) - finally: - set_version(False) - # Delete the spec file - we're good without. - os.remove("{}.spec".format(tool)) - - # Test if it works at all O:-) - executable = join(PYINSTALLER_DIST, tool) - if platform.system() == "Windows": - executable += ".exe" - - # Remove _main suffix from mitmproxy executable - if "_main" in executable: - shutil.move( - executable, - executable.replace("_main", "") - ) - executable = executable.replace("_main", "") - - print("> %s --version" % executable) - print(subprocess.check_output([executable, "--version"]).decode()) - - archive.add(executable, basename(executable)) - print("Packed {}.".format(archive_name(bdist))) - - @cli.command("upload-release") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @@ -305,77 +101,13 @@ def homebrew_pr(): ]) -@cli.command("upload-snapshot") -@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) -@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) -@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) -@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) -@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) -@click.option("--wheel/--no-wheel", default=False) -@click.option("--bdist/--no-bdist", default=False) -@click.option("--installer/--no-installer", default=False) -def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): - """ - Upload snapshot to snapshot server - """ - cnopts = pysftp.CnOpts( - knownhosts=join(RELEASE_DIR, 'known_hosts') - ) - with pysftp.Connection(host=host, - port=port, - username=user, - private_key=private_key, - private_key_pass=private_key_password, - cnopts=cnopts) as sftp: - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - if bdist: - for bdist in sorted(BDISTS.keys()): - files.append(archive_name(bdist)) - if installer: - files.append(installer_name()) - - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = re.sub( - r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), - get_version(True, True), - f - ) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - # We hide the file during upload - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Show new version - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - if f != wheel_name(): - # "latest" isn't a proper wheel version, so this could not be installed. - # https://github.com/mitmproxy/mitmproxy/issues/1065 - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) if __name__ == "__main__": @@ -88,7 +88,7 @@ setup( 'dev': [ "flake8>=3.5, <3.6", "Flask>=0.10.1, <0.13", - "mypy>=0.560,<0.561", + "mypy>=0.570,<0.571", "pytest-cov>=2.5.1,<3", "pytest-faulthandler>=1.3.1,<2", "pytest-timeout>=1.2.1,<2", diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index ce49002c..ef581a91 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -351,10 +351,10 @@ class TestRequestUtils: request.headers["Content-Type"] = "application/x-www-form-urlencoded" assert list(request.urlencoded_form.items()) == [("foobar", "baz")] request.raw_content = b"\xFF" - assert len(request.urlencoded_form) == 0 + assert len(request.urlencoded_form) == 1 def test_set_urlencoded_form(self): - request = treq() + request = treq(content=b"\xec\xed") request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')] assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" assert request.content diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py index db8a63a7..108f238e 100644 --- a/test/mitmproxy/tools/console/test_statusbar.py +++ b/test/mitmproxy/tools/console/test_statusbar.py @@ -1,3 +1,5 @@ +import pytest + from mitmproxy import options from mitmproxy.tools.console import statusbar, master @@ -31,3 +33,29 @@ def test_statusbar(monkeypatch): bar = statusbar.StatusBar(m) # this already causes a redraw assert bar.ib._w + + +@pytest.mark.parametrize("message,ready_message", [ + ("", [(None, ""), ("warn", "")]), + (("info", "Line fits into statusbar"), [("info", "Line fits into statusbar"), + ("warn", "")]), + ("Line doesn't fit into statusbar", [(None, "Line doesn'\u2026"), + ("warn", "(more in eventlog)")]), + (("alert", "Two lines.\nFirst fits"), [("alert", "Two lines."), + ("warn", "(more in eventlog)")]), + ("Two long lines\nFirst doesn't fit", [(None, "Two long li\u2026"), + ("warn", "(more in eventlog)")]) +]) +def test_shorten_message(message, ready_message): + o = options.Options() + m = master.ConsoleMaster(o) + ab = statusbar.ActionBar(m) + assert ab.shorten_message(message, max_width=30) == ready_message + + +def test_shorten_message_narrow(): + o = options.Options() + m = master.ConsoleMaster(o) + ab = statusbar.ActionBar(m) + shorten_msg = ab.shorten_message("error", max_width=4) + assert shorten_msg == [(None, "\u2026"), ("warn", "(more in eventlog)")] @@ -32,27 +32,12 @@ deps = commands = python test/individual_coverage.py -[testenv:wheel] -recreate = True -deps = -commands = - python setup.py -q bdist_wheel --dist-dir release/dist - pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl - # skip `mitmproxy --version` if SKIP_MITMPROXY is defined. - {env:SKIP_MITMPROXY:mitmproxy --version} - mitmdump --version - mitmweb --version - pathod --version - pathoc --version - -[testenv:rtool] -passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY +[testenv:cibuild] +passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps = -rrequirements.txt pyinstaller==3.3.1 - twine==1.9.1 - pysftp==0.2.9 - + awscli commands = mitmdump --version - python ./release/rtool.py {posargs} + python ./release/ci.py {posargs} diff --git a/web/src/js/components/Header/FilterDocs.jsx b/web/src/js/components/Header/FilterDocs.jsx index b471840b..198c0117 100644 --- a/web/src/js/components/Header/FilterDocs.jsx +++ b/web/src/js/components/Header/FilterDocs.jsx @@ -44,7 +44,7 @@ export default class FilterDocs extends Component { ))} <tr key="docs-link"> <td colSpan="2"> - <a href="http://docs.mitmproxy.org/en/stable/features/filters.html" + <a href="https://mitmproxy.org/docs/latest/concepts-filters/" target="_blank"> <i className="fa fa-external-link"></i> mitmproxy docs</a> diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index e4b151ad..80f43018 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -253,7 +253,7 @@ BindingExpr / Expr /* All the filters except "~s" and "~src" are arranged in the ascending order as - given in the docs(http://docs.mitmproxy.org/en/latest/features/filters.html). + given in the docs(https://mitmproxy.org/docs/latest/concepts-filters/). "~s" and "~src" are so arranged as "~s" caused problems in the evaluation of "~src". */ |