diff options
35 files changed, 706 insertions, 377 deletions
diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index e9c62ba1..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: '{build}' -build: off # Not a C# project - -branches: - except: - - requires-io-master - -environment: - CI_DEPS: codecov>=2.0.5 - CI_COMMANDS: codecov - matrix: - - PYTHON: "C:\\Python36" - TOXENV: "py36" - PYINSTALLER: "1" - WININSTALLER: "1" - - PYTHON: "C:\\Python37" - TOXENV: "py37" - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "python -m pip install --disable-pip-version-check -U pip" - - "pip install -U tox" - -test_script: - - ps: | - if ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[notest]")) { - echo "!!!! Skipping tests." - } else { - tox -- --verbose --cov-report=term - } - - ps: tox -e cibuild -- build - -deploy_script: - ps: tox -e cibuild -- upload - -cache: - - C:\projects\mitmproxy\release\installbuilder\setup -> .appveyor.yml - - C:\Users\appveyor\AppData\Local\pip\cache diff --git a/.gitattributes b/.gitattributes index 69d68b8e..7e6d229f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ mitmproxy/tools/web/static/**/* -diff linguist-vendored web/src/js/filt/filt.js -diff +*.bin binary
\ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..c9f83e4e --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,210 @@ +name: CI + +on: [push, pull_request] + +# We currently use Python 3.7 for most things: +# - zstandard currently doesn't have 3.8 wheels, +# - we need to upgrade cryptography from version 2.4, which also doesn't have wheels + +env: + # Codecov + CODECOV_TOKEN: "0409bdfd-57a4-477d-a8af-f6172ef431d3" + +jobs: + lint-pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: TrueBrain/actions-flake8@v1.2 + lint-local: + # do not use external action when secrets are exposed. + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install flake8 + - run: flake8 mitmproxy pathod examples test release + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install mypy + - run: mypy . + test: + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - run: printenv + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: tox -e py37 + # codecov's GitHub action only supports Linux. https://github.com/codecov/codecov-action/issues/7 + # codecov's Python uploader has no github actions support yet. https://github.com/codecov/codecov-python/pull/214 + - name: Extract branch name # https://stackoverflow.com/a/58035262/934719 + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - run: pip install codecov + - run: > + codecov -f coverage.xml + --name python-${{ matrix.os }} + --commit ${{ github.sha }} + --slug ${{ github.repository }} + --branch ${{ steps.extract_branch.outputs.branch }} + test-py35: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.5' + - run: pip install tox + - run: tox -e py35 + build-wheel: + runs-on: ubuntu-latest + env: + CI_BUILD_WHEEL: 1 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: tox -e cibuild -- build + - uses: actions/upload-artifact@master + with: + name: wheel + path: release/dist + build-binaries: + strategy: + fail-fast: false + matrix: + # Old Ubuntu version for old glibc + os: [macos-latest, windows-latest, ubuntu-16.04] + runs-on: ${{ matrix.os }} + env: + CI_BUILD_PYINSTALLER: 1 + CI_BUILD_WININSTALLER: ${{ matrix.os == 'windows-latest' }} + CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - if: matrix.os == 'windows-latest' + uses: actions/cache@v1 + with: + path: release/installbuilder/setup + key: installbuilder + - run: pip install tox + - run: tox -e cibuild -- build + # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24 + - uses: actions/upload-artifact@master + with: + name: binaries.${{ matrix.os }} + path: release/dist + + test-web-ui: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: git rev-parse --abbrev-ref HEAD + - uses: actions/setup-node@v1 + - id: yarn-cache + run: echo "::set-output name=dir::$(yarn cache dir)" + - uses: actions/cache@v1 + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - working-directory: ./web + run: yarn + - working-directory: ./web + run: npm test + - run: bash <(curl -s https://codecov.io/bash) + + docs: + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: | + wget https://github.com/gohugoio/hugo/releases/download/v0.59.1/hugo_0.59.1_Linux-64bit.deb + sudo dpkg -i hugo*.deb + - run: tox -e docs + + # Separate from everything else because slow. + build-and-deploy-docker: + if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' + needs: [test, test-web-ui, build-wheel] + runs-on: ubuntu-latest + env: + CI_BUILD_DOCKER: 1 + DOCKER_USERNAME: mitmbot + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - uses: actions/download-artifact@master + with: + name: wheel + path: release/dist + - run: tox -e cibuild -- build + - run: tox -e cibuild -- upload + + deploy: + if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' + runs-on: ubuntu-latest + needs: [test, test-web-ui, build-wheel, build-binaries] + env: + CI_BUILD_WHEEL: 1 + CI_BUILD_PYINSTALLER: 1 + CI_BUILD_WININSTALLER: 1 + TWINE_USERNAME: mitmproxy + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + # artifacts must be downloaded individually, see https://github.com/actions/download-artifact/issues/6 + - uses: actions/download-artifact@master + with: + name: wheel + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.windows-latest + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.macos-latest + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.ubuntu-16.04 + path: release/dist + - run: ls release/dist + - run: pip install tox + - run: tox -e cibuild -- upload diff --git a/.landscape.yml b/.landscape.yml deleted file mode 100644 index b6a45ed7..00000000 --- a/.landscape.yml +++ /dev/null @@ -1,20 +0,0 @@ -ignore-paths: - - docs - - examples - - mitmproxy/contrib - - web -max-line-length: 140 -pylint: - options: - dummy-variables-rgx: _$|.+_$|dummy_.+ - disable: - - missing-docstring - - protected-access - - too-few-public-methods - - too-many-arguments - - too-many-instance-attributes - - too-many-locals - - too-many-public-methods - - too-many-return-statements - - too-many-statements - - unpacking-non-sequence diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 035efb79..00000000 --- a/.travis.yml +++ /dev/null @@ -1,103 +0,0 @@ -language: python - -branches: - except: - - requires-io-master - -env: - global: - - CI_DEPS=codecov>=2.0.5 - - CI_COMMANDS=codecov - -git: - depth: 10000 - -matrix: - fast_finish: true - include: - - python: 3.5 - env: TOXENV=py35 # this just makes sure that our version detection shows an appropriate error message - - python: 3.6 - env: TOXENV=lint - - os: osx - osx_image: xcode7.3 - language: generic - env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 - - python: 3.6 - env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 WHEEL=1 DOCKER=1 - sudo: required - services: - - docker - - python: 3.6 - env: TOXENV=individual_coverage - - python: 3.7 - env: TOXENV=py37 - dist: xenial - - language: node_js - node_js: "node" - before_install: - - curl -o- -L https://yarnpkg.com/install.sh | bash - - export PATH=$HOME/.yarn/bin:$PATH - install: - - cd web && yarn - - yarn global add codecov - script: npm test && codecov - cache: - yarn: true - directories: - - web/node_modules - - python: 3.6 - env: NAME=docs TOXENV=docs - install: - - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - - sudo dpkg -i hugo*.deb - - pip install -U tox virtualenv setuptools - script: - - tox - after_success: - - echo done - -install: - - | - if [[ $TRAVIS_OS_NAME == "osx" ]] - then - brew update || brew update - brew outdated pyenv || brew upgrade pyenv - eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9 - pyenv global 3.6.9 - pyenv shell 3.6.9 - fi - - pip install -U tox virtualenv setuptools - -script: - # All these steps MUST succeed for the job to be successful! - # Using the after_success block DOES NOT capture errors. - # Pull requests might break our build - we need to check this. - # Pull requests are not allowed to upload build artifacts - enforced by cibuild script. - - | - if [[ $TRAVIS_COMMIT_MESSAGE = *"[notest]"* ]]; then - echo "!!!! Skipping tests." - else - tox -- --verbose --cov-report=term - fi - - - | - if [[ $CIBUILD == "1" ]] - then - git fetch --unshallow --tags - tox -e cibuild -- build && tox -e cibuild -- upload - fi - -notifications: - slack: - -rooms: - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu - on_success: change - on_failure: change - on_start: never - -cache: - directories: - - $HOME/.pyenv - - $HOME/.cache/pip @@ -1,7 +1,7 @@ mitmproxy ^^^^^^^^^ -|travis| |appveyor| |coverage| |latest_release| |python_versions| +|ci_status| |coverage| |latest_release| |python_versions| This repository contains the **mitmproxy** and **pathod** projects. @@ -163,13 +163,9 @@ with the following command: :target: http://slack.mitmproxy.org/ :alt: Slack Developer Chat -.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg?label=travis%20ci - :target: https://travis-ci.org/mitmproxy/mitmproxy - :alt: Travis Build Status - -.. |appveyor| image:: https://shields.mitmproxy.org/appveyor/ci/mitmproxy/mitmproxy/master.svg?label=appveyor%20ci - :target: https://ci.appveyor.com/project/mitmproxy/mitmproxy - :alt: Appveyor Build Status +.. |ci_status| image:: https://github.com/mitmproxy/mitmproxy/workflows/CI/badge.svg?branch=master + :target: https://github.com/mitmproxy/mitmproxy/actions?query=branch%3Amaster + :alt: Continuous Integration Status .. |coverage| image:: https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/master.svg?label=codecov :target: https://codecov.io/gh/mitmproxy/mitmproxy @@ -7,7 +7,7 @@ set -e # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). -if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $GITHUB_REF == "refs/heads/master" ]]; then aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ --paths "/master/*" diff --git a/examples/complex/change_upstream_proxy.py b/examples/complex/change_upstream_proxy.py index 089a9df5..a0e7e572 100644 --- a/examples/complex/change_upstream_proxy.py +++ b/examples/complex/change_upstream_proxy.py @@ -24,4 +24,4 @@ def request(flow: http.HTTPFlow) -> None: return address = proxy_address(flow) if flow.live: - flow.live.change_upstream_proxy_server(address) + flow.live.change_upstream_proxy_server(address) # type: ignore diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index 69b9ea9e..8b904216 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -31,6 +31,7 @@ def request(flow: http.HTTPFlow) -> None: def response(flow: http.HTTPFlow) -> None: + assert flow.response flow.response.headers.pop('Strict-Transport-Security', None) flow.response.headers.pop('Public-Key-Pins', None) diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index d5f4aaab..2a45511a 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -395,8 +395,10 @@ def get_XSS_data(body: Union[str, bytes], request_URL: str, injection_point: str # response is mitmproxy's entry point def response(flow: http.HTTPFlow) -> None: + assert flow.response cookies_dict = get_cookies(flow) resp = flow.response.get_text(strict=False) + assert resp # Example: http://xss.guru/unclaimedScriptTag.html find_unclaimed_URLs(resp, flow.request.url) results = test_end_of_URL_injection(resp, flow.request.url, cookies_dict) diff --git a/issue_template.md b/issue_template.md deleted file mode 100644 index 3dbac2ac..00000000 --- a/issue_template.md +++ /dev/null @@ -1,17 +0,0 @@ -##### Steps to reproduce the problem: - -1. -2. -3. - - -##### Any other comments? What have you tried so far? - - - -##### System information - -<!-- Paste the output of "mitmproxy --version" here. --> - - -<!-- Please use StackOverflow (https://stackoverflow.com/questions/tagged/mitmproxy) for support/how-to questions. Thanks! :) --> diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 7bdaeb33..7adefd7a 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,23 +1,23 @@ import queue import threading -import typing import time +import typing -from mitmproxy import log +import mitmproxy.types +from mitmproxy import command +from mitmproxy import connections from mitmproxy import controller +from mitmproxy import ctx from mitmproxy import exceptions -from mitmproxy import http from mitmproxy import flow +from mitmproxy import http +from mitmproxy import io +from mitmproxy import log from mitmproxy import options -from mitmproxy import connections +from mitmproxy.coretypes import basethread from mitmproxy.net import server_spec, tls from mitmproxy.net.http import http1 -from mitmproxy.coretypes import basethread from mitmproxy.utils import human -from mitmproxy import ctx -from mitmproxy import io -from mitmproxy import command -import mitmproxy.types class RequestReplayThread(basethread.BaseThread): @@ -117,7 +117,7 @@ class RequestReplayThread(basethread.BaseThread): finally: r.first_line_format = first_line_format_backup f.live = False - if server.connected(): + if server and server.connected(): server.finish() server.close() diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 2776118a..68df9374 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -1,66 +1,114 @@ +import shlex import typing -from mitmproxy import ctx +import pyperclip + +import mitmproxy.types from mitmproxy import command -from mitmproxy import flow +from mitmproxy import ctx, http from mitmproxy import exceptions -from mitmproxy.utils import strutils +from mitmproxy import flow from mitmproxy.net.http.http1 import assemble -import mitmproxy.types - -import pyperclip +from mitmproxy.utils import strutils -def cleanup_request(f: flow.Flow): - if not hasattr(f, "request"): +def cleanup_request(f: flow.Flow) -> http.HTTPRequest: + if not getattr(f, "request", None): raise exceptions.CommandError("Can't export flow with no request.") - request = f.request.copy() # type: ignore + assert isinstance(f, http.HTTPFlow) + request = f.request.copy() request.decode(strict=False) - # a bit of clean-up - if request.method == 'GET' and request.headers.get("content-length", None) == "0": - request.headers.pop('content-length') - request.headers.pop(':authority', None) + # a bit of clean-up - these headers should be automatically set by curl/httpie + request.headers.pop('content-length') + if request.headers.get("host", "") == request.host: + request.headers.pop("host") + if request.headers.get(":authority", "") == request.host: + request.headers.pop(":authority") return request +def cleanup_response(f: flow.Flow) -> http.HTTPResponse: + if not getattr(f, "response", None): + raise exceptions.CommandError("Can't export flow with no response.") + assert isinstance(f, http.HTTPFlow) + response = f.response.copy() # type: ignore + response.decode(strict=False) + return response + + +def request_content_for_console(request: http.HTTPRequest) -> str: + try: + text = request.get_text(strict=True) + assert text + except ValueError: + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + raise exceptions.CommandError("Request content must be valid unicode") + escape_control_chars = {chr(i): f"\\x{i:02x}" for i in range(32)} + return "".join( + escape_control_chars.get(x, x) + for x in text + ) + + def curl_command(f: flow.Flow) -> str: - data = "curl " request = cleanup_request(f) + args = ["curl"] for k, v in request.headers.items(multi=True): - data += "--compressed " if k == 'accept-encoding' else "" - data += "-H '%s:%s' " % (k, v) + if k.lower() == "accept-encoding": + args.append("--compressed") + else: + args += ["-H", f"{k}: {v}"] + if request.method != "GET": - data += "-X %s " % request.method - data += "'%s'" % request.url + args += ["-X", request.method] + args.append(request.url) if request.content: - data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( - request.content, - escape_single_quotes=True - ) - return data + args += ["-d", request_content_for_console(request)] + return ' '.join(shlex.quote(arg) for arg in args) def httpie_command(f: flow.Flow) -> str: request = cleanup_request(f) - data = "http %s %s" % (request.method, request.url) + args = ["http", request.method, request.url] for k, v in request.headers.items(multi=True): - data += " '%s:%s'" % (k, v) + args.append(f"{k}: {v}") + cmd = ' '.join(shlex.quote(arg) for arg in args) if request.content: - data += " <<< '%s'" % strutils.bytes_to_escaped_str( - request.content, - escape_single_quotes=True - ) - return data + cmd += " <<< " + shlex.quote(request_content_for_console(request)) + return cmd + + +def raw_request(f: flow.Flow) -> bytes: + return assemble.assemble_request(cleanup_request(f)) + + +def raw_response(f: flow.Flow) -> bytes: + return assemble.assemble_response(cleanup_response(f)) + + +def raw(f: flow.Flow, separator=b"\r\n\r\n") -> bytes: + """Return either the request or response if only one exists, otherwise return both""" + request_present = hasattr(f, "request") and f.request # type: ignore + response_present = hasattr(f, "response") and f.response # type: ignore + if not (request_present or response_present): + raise exceptions.CommandError("Can't export flow with no request or response.") -def raw(f: flow.Flow) -> bytes: - return assemble.assemble_request(cleanup_request(f)) # type: ignore + if request_present and response_present: + return b"".join([raw_request(f), separator, raw_response(f)]) + elif not request_present: + return raw_response(f) + else: + return raw_request(f) formats = dict( - curl = curl_command, - httpie = httpie_command, - raw = raw, + curl=curl_command, + httpie=httpie_command, + raw=raw, + raw_request=raw_request, + raw_response=raw_response, ) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 18bc3545..7f642585 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,16 +1,19 @@ import hashlib -import urllib import typing +import urllib -from mitmproxy import ctx -from mitmproxy import flow +import mitmproxy.types +from mitmproxy import command +from mitmproxy import ctx, http from mitmproxy import exceptions +from mitmproxy import flow from mitmproxy import io -from mitmproxy import command -import mitmproxy.types class ServerPlayback: + flowmap: typing.Dict[typing.Hashable, typing.List[http.HTTPFlow]] + configured: bool + def __init__(self): self.flowmap = {} self.configured = False @@ -82,10 +85,10 @@ class ServerPlayback: Replay server responses from flows. """ self.flowmap = {} - for i in flows: - if i.response: # type: ignore - l = self.flowmap.setdefault(self._hash(i), []) - l.append(i) + for f in flows: + if isinstance(f, http.HTTPFlow): + lst = self.flowmap.setdefault(self._hash(f), []) + lst.append(f) ctx.master.addons.trigger("update", []) @command.command("replay.server.file") @@ -108,12 +111,11 @@ class ServerPlayback: def count(self) -> int: return sum([len(i) for i in self.flowmap.values()]) - def _hash(self, flow): + def _hash(self, flow: http.HTTPFlow) -> typing.Hashable: """ Calculates a loose hash of the flow request. """ r = flow.request - _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) @@ -158,20 +160,32 @@ class ServerPlayback: repr(key).encode("utf8", "surrogateescape") ).digest() - def next_flow(self, request): + def next_flow(self, flow: http.HTTPFlow) -> typing.Optional[http.HTTPFlow]: """ Returns the next flow object, or None if no matching flow was found. """ - hsh = self._hash(request) - if hsh in self.flowmap: + hash = self._hash(flow) + if hash in self.flowmap: if ctx.options.server_replay_nopop: - return self.flowmap[hsh][0] + return next(( + flow + for flow in self.flowmap[hash] + if flow.response + ), None) else: - ret = self.flowmap[hsh].pop(0) - if not self.flowmap[hsh]: - del self.flowmap[hsh] + ret = self.flowmap[hash].pop(0) + while not ret.response: + if self.flowmap[hash]: + ret = self.flowmap[hash].pop(0) + else: + del self.flowmap[hash] + return None + if not self.flowmap[hash]: + del self.flowmap[hash] return ret + else: + return None def configure(self, updated): if not self.configured and ctx.options.server_replay: @@ -182,10 +196,11 @@ class ServerPlayback: raise exceptions.OptionsError(str(e)) self.load_flows(flows) - def request(self, f): + def request(self, f: http.HTTPFlow) -> None: if self.flowmap: rflow = self.next_flow(f) if rflow: + assert rflow.response response = rflow.response.copy() response.is_replay = True if ctx.options.server_replay_refresh: @@ -197,4 +212,5 @@ class ServerPlayback: f.request.url ) ) + assert f.reply f.reply.kill() diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index 2496d47c..07cee466 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -558,7 +558,7 @@ class TCPServer: self.socket = None try: - # First try to bind an IPv6 socket, with possible IPv4 if the OS supports it. + # First try to bind an IPv6 socket, attempting to enable IPv4 support if the OS supports it. # This allows us to accept connections for ::1 and 127.0.0.1 on the same socket. # Only works if self.address == "" self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) @@ -572,8 +572,20 @@ class TCPServer: self.socket = None if not self.socket: - # Binding to an IPv6 socket failed, lets fall back to IPv4. - self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # Binding to an IPv6 + IPv4 socket failed, lets fall back to IPv4 only. + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + self.socket.bind(self.address) + except socket.error: + if self.socket: + self.socket.close() + self.socket = None + + if not self.socket: + # Binding to an IPv4 only socket failed, lets fall back to IPv6 only. + self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) self.socket.bind(self.address) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index a00a3e98..0163e8d3 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -114,7 +114,7 @@ def run( loop = asyncio.get_event_loop() for signame in ('SIGINT', 'SIGTERM'): try: - loop.add_signal_handler(getattr(signal, signame), master.shutdown) + loop.add_signal_handler(getattr(signal, signame), getattr(master, "prompt_for_exit", master.shutdown)) except NotImplementedError: # Not supported on Windows pass diff --git a/release/cibuild.py b/release/cibuild.py index 82060639..46066099 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -14,9 +14,10 @@ import urllib.request import zipfile import click -import cryptography.fernet import parver +import cryptography.fernet + @contextlib.contextmanager def chdir(path: str): # pragma: no cover @@ -30,6 +31,14 @@ class BuildError(Exception): pass +def bool_from_env(envvar: str) -> bool: + val = os.environ.get(envvar, "") + if not val or val.lower() in ("0", "false"): + return False + else: + return True + + class BuildEnviron: PLATFORM_TAGS = { "Darwin": "osx", @@ -38,25 +47,27 @@ class BuildEnviron: } def __init__( - self, - *, - system="", - root_dir="", - travis_tag="", - travis_branch="", - travis_pull_request="", - appveyor_repo_tag_name="", - appveyor_repo_branch="", - appveyor_pull_request_number="", - should_build_wheel=False, - should_build_docker=False, - should_build_pyinstaller=False, - should_build_wininstaller=False, - has_aws_creds=False, - has_twine_creds=False, - docker_username="", - docker_password="", - rtool_key="", + self, + *, + system="", + root_dir="", + travis_tag="", + travis_branch="", + travis_pull_request="", + appveyor_repo_tag_name="", + appveyor_repo_branch="", + appveyor_pull_request_number="", + github_ref="", + github_event_name="", + should_build_wheel=False, + should_build_docker=False, + should_build_pyinstaller=False, + should_build_wininstaller=False, + has_aws_creds=False, + has_twine_creds=False, + docker_username="", + docker_password="", + build_key="", ): self.system = system self.root_dir = root_dir @@ -80,11 +91,14 @@ class BuildEnviron: self.appveyor_repo_branch = appveyor_repo_branch self.appveyor_pull_request_number = appveyor_pull_request_number + self.github_ref = github_ref + self.github_event_name = github_event_name + self.has_aws_creds = has_aws_creds self.has_twine_creds = has_twine_creds self.docker_username = docker_username self.docker_password = docker_password - self.rtool_key = rtool_key + self.build_key = build_key @classmethod def from_env(cls): @@ -96,19 +110,18 @@ class BuildEnviron: travis_pull_request=os.environ.get("TRAVIS_PULL_REQUEST"), appveyor_repo_tag_name=os.environ.get("APPVEYOR_REPO_TAG_NAME", ""), appveyor_repo_branch=os.environ.get("APPVEYOR_REPO_BRANCH", ""), - appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), - should_build_wheel="WHEEL" in os.environ, - should_build_pyinstaller="PYINSTALLER" in os.environ, - should_build_wininstaller="WININSTALLER" in os.environ, - should_build_docker="DOCKER" in os.environ, - has_aws_creds="AWS_ACCESS_KEY_ID" in os.environ, - has_twine_creds=( - "TWINE_USERNAME" in os.environ and - "TWINE_PASSWORD" in os.environ - ), - docker_username=os.environ.get("DOCKER_USERNAME"), - docker_password=os.environ.get("DOCKER_PASSWORD"), - rtool_key=os.environ.get("RTOOL_KEY"), + appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER", ""), + github_ref=os.environ.get("GITHUB_REF", ""), + github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""), + should_build_wheel=bool_from_env("CI_BUILD_WHEEL"), + should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"), + should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"), + should_build_docker=bool_from_env("CI_BUILD_DOCKER"), + has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"), + has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"), + docker_username=os.environ.get("DOCKER_USERNAME", ""), + docker_password=os.environ.get("DOCKER_PASSWORD", ""), + build_key=os.environ.get("CI_BUILD_KEY", ""), ) def archive(self, path): @@ -143,26 +156,34 @@ class BuildEnviron: return ret @property - def branch(self): - return self.travis_branch or self.appveyor_repo_branch + def branch(self) -> str: + if self.travis_branch: + return self.travis_branch + if self.appveyor_repo_branch: + return self.appveyor_repo_branch + if self.github_ref and self.github_ref.startswith("refs/heads/"): + return self.github_ref.replace("refs/heads/", "") + if self.github_ref and self.github_ref.startswith("refs/pull/"): + return "pr-" + self.github_ref.split("/")[2] + return "" @property - def build_dir(self): + def build_dir(self) -> str: return os.path.join(self.release_dir, "build") @property - def dist_dir(self): + def dist_dir(self) -> str: return os.path.join(self.release_dir, "dist") @property - def docker_tag(self): + def docker_tag(self) -> str: if self.branch == "master": t = "dev" else: t = self.version return "mitmproxy/mitmproxy:{}".format(t) - def dump_info(self, fp=sys.stdout): + def dump_info(self, fp=sys.stdout) -> None: lst = [ "version", "tag", @@ -176,7 +197,9 @@ class BuildEnviron: "upload_dir", "should_build_wheel", "should_build_pyinstaller", + "should_build_wininstaller", "should_build_docker", + "should_upload_aws", "should_upload_docker", "should_upload_pypi", ] @@ -190,7 +213,9 @@ class BuildEnviron: """ with open(pathlib.Path(self.root_dir) / "mitmproxy" / "version.py") as f: contents = f.read() - version = re.search(r'^VERSION = "(.+?)"', contents, re.M).group(1) + match = re.search(r'^VERSION = "(.+?)"', contents, re.M) + assert match + version = match.group(1) if self.is_prod_release: # For production releases, we require strict version equality @@ -230,6 +255,8 @@ class BuildEnviron: @property def is_pull_request(self) -> bool: + if self.github_event_name == "pull_request": + return True if self.appveyor_pull_request_number: return True if self.travis_pull_request and self.travis_pull_request != "false": @@ -237,13 +264,13 @@ class BuildEnviron: return False @property - def platform_tag(self): + def platform_tag(self) -> str: if self.system in self.PLATFORM_TAGS: return self.PLATFORM_TAGS[self.system] raise BuildError("Unsupported platform: %s" % self.system) @property - def release_dir(self): + def release_dir(self) -> str: return os.path.join(self.root_dir, "release") @property @@ -255,6 +282,13 @@ class BuildEnviron: ]) @property + def should_upload_aws(self) -> bool: + return all([ + self.has_aws_creds, + (self.should_build_wheel or self.should_build_pyinstaller or self.should_build_wininstaller), + ]) + + @property def should_upload_pypi(self) -> bool: return all([ self.is_prod_release, @@ -263,18 +297,24 @@ class BuildEnviron: ]) @property - def tag(self): - return self.travis_tag or self.appveyor_repo_tag_name + def tag(self) -> str: + if self.travis_tag: + return self.travis_tag + if self.appveyor_repo_tag_name: + return self.appveyor_repo_tag_name + if self.github_ref and self.github_ref.startswith("refs/tags/"): + return self.github_ref.replace("refs/tags/", "") + return "" @property - def upload_dir(self): + def upload_dir(self) -> str: if self.tag: return self.version else: return "branches/%s" % self.version @property - def version(self): + def version(self) -> str: if self.tag: if self.tag.startswith("v"): try: @@ -298,13 +338,14 @@ def build_wheel(be: BuildEnviron): # pragma: no cover "bdist_wheel", "--dist-dir", be.dist_dir, ]) - whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] + whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) click.echo("Found wheel package: {}".format(whl)) subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) return whl -def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover +def build_docker_image(be: BuildEnviron): # pragma: no cover + whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) click.echo("Building Docker images...") subprocess.check_call([ "docker", @@ -408,22 +449,25 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover def build_wininstaller(be: BuildEnviron): # pragma: no cover + if not be.build_key: + click.echo("Cannot build windows installer without secret key.") + return click.echo("Building wininstaller package...") - IB_VERSION = "18.8.0" + IB_VERSION = "19.10.0" IB_DIR = pathlib.Path(be.release_dir) / "installbuilder" IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe" IB_CLI = fr"C:\Program Files (x86)\BitRock InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe" IB_LICENSE = IB_DIR / "license.xml" - if True or not os.path.isfile(IB_CLI): + if not os.path.isfile(IB_CLI): if not os.path.isfile(IB_SETUP): click.echo("Downloading InstallBuilder...") def report(block, blocksize, total): done = block * blocksize if round(100 * done / total) != round(100 * (done - blocksize) / total): - click.secho(f"Downloading... {round(100*done/total)}%") + click.secho(f"Downloading... {round(100 * done / total)}%") urllib.request.urlretrieve( f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe", @@ -433,14 +477,13 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover shutil.move(IB_SETUP.with_suffix(".tmp"), IB_SETUP) click.echo("Install InstallBuilder...") - subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], - check=True) + subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True) assert os.path.isfile(IB_CLI) click.echo("Decrypt InstallBuilder license...") - f = cryptography.fernet.Fernet(be.rtool_key.encode()) - with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, open(IB_LICENSE, - "wb") as outfile: + f = cryptography.fernet.Fernet(be.build_key.encode()) + with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \ + open(IB_LICENSE, "wb") as outfile: outfile.write(f.decrypt(infile.read())) click.echo("Run InstallBuilder...") @@ -477,13 +520,12 @@ def build(): # pragma: no cover os.makedirs(be.dist_dir, exist_ok=True) if be.should_build_wheel: - whl = build_wheel(be) - # Docker image requires wheels - if be.should_build_docker: - build_docker_image(be, whl) + build_wheel(be) + if be.should_build_docker: + build_docker_image(be) if be.should_build_pyinstaller: build_pyinstaller(be) - if be.should_build_wininstaller and be.rtool_key: + if be.should_build_wininstaller: build_wininstaller(be) @@ -497,12 +539,15 @@ def upload(): # pragma: no cover Pushes the Docker image to Docker Hub. """ be = BuildEnviron.from_env() + be.dump_info() if be.is_pull_request: click.echo("Refusing to upload artifacts from a pull request!") return - if be.has_aws_creds: + if be.should_upload_aws: + num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)]) + click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...") subprocess.check_call([ "aws", "s3", "cp", "--acl", "public-read", @@ -19,12 +19,18 @@ exclude_lines = pragma: no cover raise NotImplementedError() +[mypy] +ignore_missing_imports = True + [mypy-mitmproxy.contrib.*] ignore_errors = True [mypy-tornado.*] ignore_errors = True +[mypy-test.*] +ignore_errors = True + [tool:full_coverage] exclude = mitmproxy/proxy/protocol/base.py @@ -13,7 +13,9 @@ with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() with open(os.path.join(here, "mitmproxy", "version.py")) as f: - VERSION = re.search(r'VERSION = "(.+?)"', f.read()).group(1) + match = re.search(r'VERSION = "(.+?)"', f.read()) + assert match + VERSION = match.group(1) setup( name="mitmproxy", @@ -74,7 +76,7 @@ setup( "passlib>=1.6.5, <1.8", "protobuf>=3.6.0, <3.11", "pyasn1>=0.3.1,<0.5", - "pyOpenSSL>=19.0.0,<20", + "pyOpenSSL==19.0.0", "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.8", "ruamel.yaml>=0.16,<0.17", diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index c86e0c7d..b0e5e47e 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -1,4 +1,5 @@ import os +import shlex import pytest import pyperclip @@ -18,6 +19,19 @@ def get_request(): @pytest.fixture +def get_response(): + return tflow.tflow( + resp=tutils.tresp(status_code=404, content=b"Test Response Body")) + + +@pytest.fixture +def get_flow(): + return tflow.tflow( + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz"), + resp=tutils.tresp(status_code=404, content=b"Test Response Body")) + + +@pytest.fixture def post_request(): return tflow.tflow( req=tutils.treq(method=b'POST', headers=(), content=bytes(range(256)))) @@ -41,51 +55,136 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header: qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): - result = "curl -H 'content-length:256' -X POST 'http://address:22/path' --data-binary '{}'".format( - str(bytes(range(256)))[2:-1] - ) + post_request.request.content = b'nobinarysupport' + result = "curl -X POST http://address:22/path -d nobinarysupport" assert export.curl_command(post_request) == result + def test_fails_with_binary_data(self, post_request): + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + post_request.request.headers["Content-Type"] = "application/json; charset=utf-8" + with pytest.raises(exceptions.CommandError): + export.curl_command(post_request) + def test_patch(self, patch_request): - result = """curl -H 'header:qvalue' -H 'content-length:7' -X PATCH 'http://address:22/path?query=param' --data-binary 'content'""" + result = """curl -H 'header: qvalue' -X PATCH 'http://address:22/path?query=param' -d content""" assert export.curl_command(patch_request) == result def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.curl_command(tcp_flow) + def test_escape_single_quotes_in_body(self): + request = tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=b"'&#" + ) + ) + command = export.curl_command(request) + assert shlex.split(command)[-2] == '-d' + assert shlex.split(command)[-1] == "'&#" + + def test_strip_unnecessary(self, get_request): + get_request.request.headers.clear() + get_request.request.headers["host"] = "address" + get_request.request.headers[":authority"] = "address" + get_request.request.headers["accept-encoding"] = "br" + result = """curl --compressed 'http://address:22/path?a=foo&a=bar&b=baz'""" + assert export.curl_command(get_request) == result + class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue'""" + result = """http GET 'http://address:22/path?a=foo&a=bar&b=baz' 'header: qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): - result = "http POST http://address:22/path 'content-length:256' <<< '{}'".format( - str(bytes(range(256)))[2:-1] - ) + post_request.request.content = b'nobinarysupport' + result = "http POST http://address:22/path <<< nobinarysupport" assert export.httpie_command(post_request) == result + def test_fails_with_binary_data(self, post_request): + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + post_request.request.headers["Content-Type"] = "application/json; charset=utf-8" + with pytest.raises(exceptions.CommandError): + export.httpie_command(post_request) + def test_patch(self, patch_request): - result = """http PATCH http://address:22/path?query=param 'header:qvalue' 'content-length:7' <<< 'content'""" + result = """http PATCH 'http://address:22/path?query=param' 'header: qvalue' <<< content""" assert export.httpie_command(patch_request) == result def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.httpie_command(tcp_flow) + def test_escape_single_quotes_in_body(self): + request = tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=b"'&#" + ) + ) + command = export.httpie_command(request) + assert shlex.split(command)[-2] == '<<<' + assert shlex.split(command)[-1] == "'&#" + class TestRaw: - def test_get(self, get_request): + def test_req_and_resp_present(self, get_flow): + assert b"header: qvalue" in export.raw(get_flow) + assert b"header-response: svalue" in export.raw(get_flow) + + def test_get_request_present(self, get_request): assert b"header: qvalue" in export.raw(get_request) + def test_get_response_present(self, get_response): + delattr(get_response, 'request') + assert b"header-response: svalue" in export.raw(get_response) + + def test_missing_both(self, get_request): + delattr(get_request, 'request') + delattr(get_request, 'response') + with pytest.raises(exceptions.CommandError): + export.raw(get_request) + + def test_tcp(self, tcp_flow): + with pytest.raises(exceptions.CommandError): + export.raw_request(tcp_flow) + + +class TestRawRequest: + def test_get(self, get_request): + assert b"header: qvalue" in export.raw_request(get_request) + + def test_no_request(self, get_response): + delattr(get_response, 'request') + with pytest.raises(exceptions.CommandError): + export.raw_request(get_response) + + def test_tcp(self, tcp_flow): + with pytest.raises(exceptions.CommandError): + export.raw_request(tcp_flow) + + +class TestRawResponse: + def test_get(self, get_response): + assert b"header-response: svalue" in export.raw_response(get_response) + + def test_no_response(self, get_request): + with pytest.raises(exceptions.CommandError): + export.raw_response(get_request) + def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): - export.raw(tcp_flow) + export.raw_response(tcp_flow) def qr(f): @@ -97,11 +196,15 @@ def test_export(tmpdir): f = str(tmpdir.join("path")) e = export.Export() with taddons.context(): - assert e.formats() == ["curl", "httpie", "raw"] + assert e.formats() == ["curl", "httpie", "raw", "raw_request", "raw_response"] with pytest.raises(exceptions.CommandError): e.file("nonexistent", tflow.tflow(resp=True), f) - e.file("raw", tflow.tflow(resp=True), f) + e.file("raw_request", tflow.tflow(resp=True), f) + assert qr(f) + os.unlink(f) + + e.file("raw_response", tflow.tflow(resp=True), f) assert qr(f) os.unlink(f) @@ -126,7 +229,7 @@ async def test_export_open(exception, log_message, tmpdir): with taddons.context() as tctx: with mock.patch("mitmproxy.addons.export.open") as m: m.side_effect = exception(log_message) - e.file("raw", tflow.tflow(resp=True), f) + e.file("raw_request", tflow.tflow(resp=True), f) assert await tctx.master.await_log(log_message, level="error") @@ -138,7 +241,11 @@ async def test_clip(tmpdir): e.clip("nonexistent", tflow.tflow(resp=True)) with mock.patch('pyperclip.copy') as pc: - e.clip("raw", tflow.tflow(resp=True)) + e.clip("raw_request", tflow.tflow(resp=True)) + assert pc.called + + with mock.patch('pyperclip.copy') as pc: + e.clip("raw_response", tflow.tflow(resp=True)) assert pc.called with mock.patch('pyperclip.copy') as pc: @@ -153,5 +260,5 @@ async def test_clip(tmpdir): log_message = "Pyperclip could not find a " \ "copy/paste mechanism for your system." pc.side_effect = pyperclip.PyperclipException(log_message) - e.clip("raw", tflow.tflow(resp=True)) + e.clip("raw_request", tflow.tflow(resp=True)) assert await tctx.master.await_log(log_message, level="error") diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index c6a0c1f4..2e42fa03 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -1,13 +1,13 @@ import urllib -import pytest -from mitmproxy.test import taddons -from mitmproxy.test import tflow +import pytest import mitmproxy.test.tutils -from mitmproxy.addons import serverplayback from mitmproxy import exceptions from mitmproxy import io +from mitmproxy.addons import serverplayback +from mitmproxy.test import taddons +from mitmproxy.test import tflow def tdump(path, flows): @@ -321,7 +321,7 @@ def test_server_playback_full(): with taddons.context(s) as tctx: tctx.configure( s, - server_replay_refresh = True, + server_replay_refresh=True, ) f = tflow.tflow() @@ -345,7 +345,7 @@ def test_server_playback_kill(): with taddons.context(s) as tctx: tctx.configure( s, - server_replay_refresh = True, + server_replay_refresh=True, server_replay_kill_extra=True ) @@ -357,3 +357,25 @@ def test_server_playback_kill(): f.request.host = "nonexistent" tctx.cycle(s, f) assert f.reply.value == exceptions.Kill + + +def test_server_playback_response_deleted(): + """ + The server playback addon holds references to flows that can be modified by the user in the meantime. + One thing that can happen is that users remove the response object. This happens for example when doing a client + replay at the same time. + """ + sp = serverplayback.ServerPlayback() + with taddons.context(sp) as tctx: + tctx.configure(sp) + f1 = tflow.tflow(resp=True) + f2 = tflow.tflow(resp=True) + + assert not sp.flowmap + + sp.load_flows([f1, f2]) + assert sp.flowmap + + f1.response = f2.response = None + assert not sp.next_flow(f1) + assert not sp.flowmap diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index 791045e7..f0a91fd1 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -8,7 +8,7 @@ datadir = "mitmproxy/contentviews/test_protobuf_data/" def test_view_protobuf_request(tdata): v = full_eval(protobuf.ViewProtobuf()) - p = tdata.path(datadir + "protobuf01") + p = tdata.path(datadir + "protobuf01.bin") with open(p, "rb") as f: raw = f.read() @@ -19,12 +19,12 @@ def test_view_protobuf_request(tdata): v(b'foobar') -@pytest.mark.parametrize("filename", ["protobuf02", "protobuf03"]) +@pytest.mark.parametrize("filename", ["protobuf02.bin", "protobuf03.bin"]) def test_format_pbuf(filename, tdata): path = tdata.path(datadir + filename) with open(path, "rb") as f: input = f.read() - with open(path + "-decoded") as f: + with open(path.replace(".bin", "-decoded.bin")) as f: expected = f.read() assert protobuf.format_pbuf(input) == expected diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf01 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf01.bin index fbfdbff3..fbfdbff3 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf01 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf01.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded.bin index 9be61e28..9be61e28 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02.bin Binary files differindex a47c45d5..a47c45d5 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded.bin index 3d3392e1..3d3392e1 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03.bin index 9fb230b3..9fb230b3 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03.bin diff --git a/test/mitmproxy/data/dumpfile-010 b/test/mitmproxy/data/dumpfile-010.bin Binary files differindex 435795bf..435795bf 100644 --- a/test/mitmproxy/data/dumpfile-010 +++ b/test/mitmproxy/data/dumpfile-010.bin diff --git a/test/mitmproxy/data/dumpfile-011 b/test/mitmproxy/data/dumpfile-011.bin index 936ac0cc..936ac0cc 100644 --- a/test/mitmproxy/data/dumpfile-011 +++ b/test/mitmproxy/data/dumpfile-011.bin diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018.bin index 6a27b5a6..6a27b5a6 100644 --- a/test/mitmproxy/data/dumpfile-018 +++ b/test/mitmproxy/data/dumpfile-018.bin diff --git a/test/mitmproxy/io/test_compat.py b/test/mitmproxy/io/test_compat.py index 4c31e363..341906ca 100644 --- a/test/mitmproxy/io/test_compat.py +++ b/test/mitmproxy/io/test_compat.py @@ -5,7 +5,7 @@ from mitmproxy import exceptions def test_load(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-011"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-011.bin"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -13,7 +13,7 @@ def test_load(tdata): def test_load_018(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-018"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-018.bin"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -21,7 +21,7 @@ def test_load_018(tdata): def test_cannot_convert(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-010"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-010.bin"), "rb") as f: flow_reader = io.FlowReader(f) with pytest.raises(exceptions.FlowReadException): list(flow_reader.stream()) diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 1319d1a9..38a6e1ad 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -14,7 +14,7 @@ class TestProxyConfig: def test_invalid_certificate(self, tdata): opts = options.Options() - opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] + opts.certs = [tdata.path("mitmproxy/data/dumpfile-011.bin")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index c8cf6c33..f455b0ff 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,20 +1,19 @@ import argparse import platform from unittest import mock + import pytest -from mitmproxy.tools import cmdline -from mitmproxy.tools import main from mitmproxy import options from mitmproxy.proxy import ProxyConfig -from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler from mitmproxy.proxy import config - +from mitmproxy.proxy.server import ConnectionHandler, DummyServer, ProxyServer +from mitmproxy.tools import cmdline +from mitmproxy.tools import main from ..conftest import skip_windows class MockParser(argparse.ArgumentParser): - """ argparse.ArgumentParser sys.exits() by default. Make it more testable by throwing an exception instead. @@ -53,11 +52,9 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows - @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], - reason='Skipping due to macOS Mojave') + @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-only") def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows and - # macOS Mojave + # binding to 0.0.0.0:1 works without special permissions on Windows and macOS Mojave+ conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index cfa24e63..d4ed32b0 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -58,31 +58,60 @@ def test_buildenviron_pr(): ) assert be.is_pull_request - # Mini test for appveyor - be = cibuild.BuildEnviron( - appveyor_pull_request_number="xxxx", + +def test_ci_systems(): + appveyor = cibuild.BuildEnviron( + appveyor_pull_request_number="1234", + appveyor_repo_branch="foo", + appveyor_repo_tag_name="qux", ) - assert be.is_pull_request - assert not be.is_prod_release - assert not be.is_maintenance_branch + assert appveyor.is_pull_request + assert appveyor.branch == "foo" + assert appveyor.tag == "qux" + + travis = cibuild.BuildEnviron( + travis_pull_request="1234", + travis_branch="foo", + travis_tag="foo", + ) + assert travis.is_pull_request + assert travis.branch == "foo" + assert travis.tag == "foo" + + github = cibuild.BuildEnviron( + github_event_name="pull_request", + github_ref="refs/heads/master" + ) + assert github.is_pull_request + assert github.branch == "master" + assert github.tag == "" + + github2 = cibuild.BuildEnviron( + github_event_name="pull_request", + github_ref="refs/tags/qux" + ) + assert github2.is_pull_request + assert github2.branch == "" + assert github2.tag == "qux" def test_buildenviron_commit(): # Simulates an ordinary commit on the master branch. be = cibuild.BuildEnviron( - travis_tag="", - travis_branch="master", - travis_pull_request="false", + github_ref="refs/heads/master", + github_event_name="push", should_build_wheel=True, should_build_pyinstaller=True, should_build_docker=True, docker_username="foo", docker_password="bar", + has_aws_creds=True, ) assert be.docker_tag == "mitmproxy/mitmproxy:dev" assert be.should_upload_docker assert not be.should_upload_pypi assert be.should_upload_docker + assert be.should_upload_aws assert not be.is_prod_release assert not be.is_maintenance_branch @@ -244,3 +273,20 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir): else: with pytest.raises(ValueError): be.check_version() + + +def test_bool_from_env(monkeypatch): + monkeypatch.setenv("FOO", "1") + assert cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "0") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "false") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.delenv("FOO") + assert not cibuild.bool_from_env("FOO") @@ -5,17 +5,14 @@ toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] deps = - {env:CI_DEPS:} -rrequirements.txt -passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL RTOOL_* setenv = HOME = {envtmpdir} commands = mitmdump --version - pytest --timeout 60 --cov-report='' \ + pytest --timeout 60 --cov-report xml \ --cov=mitmproxy --cov=pathod --cov=release \ --full-cov=mitmproxy/ --full-cov=pathod/ \ {posargs} - {env:CI_COMMANDS:python -c ""} [testenv:py35] whitelist_externals = @@ -31,8 +28,7 @@ commands = flake8 --jobs 8 mitmproxy pathod examples test release python ./test/filename_matching.py rstcheck README.rst - mypy --ignore-missing-imports ./mitmproxy ./pathod - mypy --ignore-missing-imports --follow-imports=skip ./examples/simple/ ./examples/pathod/ ./examples/complex/ + mypy . [testenv:individual_coverage] deps = @@ -41,7 +37,7 @@ commands = python ./test/individual_coverage.py [testenv:cibuild] -passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER +passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_* deps = -rrequirements.txt pyinstaller==3.5 @@ -63,7 +59,7 @@ commands = pathoc --version [testenv:docs] -passenv = TRAVIS_* APPVEYOR_* AWS_* +passenv = GITHUB_* AWS_* deps = -rrequirements.txt awscli |