diff options
| -rw-r--r-- | .github/ISSUE_TEMPLATE/bug_report.md | 19 | ||||
| -rw-r--r-- | .github/ISSUE_TEMPLATE/feature_request.md | 20 | ||||
| -rw-r--r-- | README.rst | 27 | ||||
| -rw-r--r-- | docs/README.md | 6 | ||||
| -rw-r--r-- | docs/src/content/howto-ignoredomains.md | 2 | ||||
| -rw-r--r-- | issue_template.md | 2 | ||||
| -rw-r--r-- | mitmproxy/addons/clientplayback.py | 5 | ||||
| -rw-r--r-- | mitmproxy/addons/export.py | 24 | ||||
| -rw-r--r-- | mitmproxy/net/http/message.py | 24 | ||||
| -rw-r--r-- | mitmproxy/net/http/request.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/response.py | 2 | ||||
| -rw-r--r-- | mitmproxy/platform/pf.py | 17 | ||||
| -rw-r--r-- | mitmproxy/utils/human.py | 4 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_export.py | 24 | ||||
| -rw-r--r-- | test/mitmproxy/data/pf01 | 6 | ||||
| -rw-r--r-- | test/mitmproxy/platform/test_pf.py | 5 | ||||
| -rw-r--r-- | test/mitmproxy/utils/test_human.py | 1 |
17 files changed, 128 insertions, 62 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..01b6fb85 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,19 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: kind/triage +assignees: '' + +--- + +#### Problem Description +A clear and concise description of what the bug is. + +#### Steps to reproduce the behavior: +1. +2. +3. + +#### System Information +Paste the output of "mitmproxy --version" here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..8e8080db --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: kind/feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +#### Describe the solution you'd like +A clear and concise description of what you want to happen. + +#### Describe alternatives you've considered +A clear and concise description of any alternative solutions or features you've considered. + +#### Additional context +Add any other context or screenshots about the feature request here. @@ -21,8 +21,7 @@ Documentation & Help -------------------- -General information, tutorials, and precompiled binaries can be found on the mitmproxy -and pathod websites. +General information, tutorials, and precompiled binaries can be found on the mitmproxy website. |mitmproxy_site| @@ -30,12 +29,10 @@ The documentation for mitmproxy is available on our website: |mitmproxy_docs_stable| |mitmproxy_docs_master| +If you have questions on how to use mitmproxy, please +ask them on StackOverflow! -Join our discussion forum on Discourse to ask questions, help -each other solve problems, and come up with new ideas for the project. - -|mitmproxy_discourse| - +|mitmproxy_stackoverflow| Join our developer chat on Slack if you would like to contribute to mitmproxy itself. @@ -54,7 +51,7 @@ Contributing As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward, please consider contributing in the following areas: -- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. +- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users on StackOverflow_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. - **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first. Development Setup @@ -146,21 +143,21 @@ with the following command: tox -e lint -.. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg +.. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg :target: https://mitmproxy.org/ :alt: mitmproxy.org -.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg +.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/badge/docs-stable-brightgreen.svg :target: https://docs.mitmproxy.org/stable/ :alt: mitmproxy documentation stable -.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg +.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/badge/docs-master-brightgreen.svg :target: https://docs.mitmproxy.org/master/ :alt: mitmproxy documentation master -.. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg - :target: https://discourse.mitmproxy.org - :alt: Discourse: mitmproxy +.. |mitmproxy_stackoverflow| image:: https://shields.mitmproxy.org/stackexchange/stackoverflow/t/mitmproxy?color=orange&label=stackoverflow%20questions + :target: https://stackoverflow.com/questions/tagged/mitmproxy + :alt: StackOverflow: mitmproxy .. |slack| image:: http://slack.mitmproxy.org/badge.svg :target: http://slack.mitmproxy.org/ @@ -195,5 +192,5 @@ with the following command: .. _yarn: https://yarnpkg.com/en/ .. _PEP8: https://www.python.org/dev/peps/pep-0008 .. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html -.. _forums: https://discourse.mitmproxy.org/ +.. _StackOverflow: https://stackoverflow.com/questions/tagged/mitmproxy .. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/docs/README.md b/docs/README.md index b39fd780..a9ee1113 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,10 +7,8 @@ This directory houses the mitmproxy documentation available at <https://docs.mit 1. Install [hugo](https://gohugo.io/). 2. Windows users: Depending on your git settings, you may need to manually create a symlink from /docs/src/examples to /examples. - -Make sure mitmproxy python package is installed. - -Run `./build-current` script in docs folder. If you skip this step, hugo will fail because files in ./src/generated are missing + 3. Make sure the mitmproxy Python package is installed. + 4. Run `./build-current` to generate the documentation source files in `./src/generated`. Now you can run `hugo server -D` in ./src. diff --git a/docs/src/content/howto-ignoredomains.md b/docs/src/content/howto-ignoredomains.md index 902a17be..9a337eba 100644 --- a/docs/src/content/howto-ignoredomains.md +++ b/docs/src/content/howto-ignoredomains.md @@ -10,7 +10,7 @@ menu: There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism: -- **Certificate pinning:** Some traffic is is protected using [Certificate +- **Certificate pinning:** Some traffic is protected using [Certificate Pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's interception leads to errors. For example, the Twitter app, Windows Update or the Apple App Store fail to work if mitmproxy is active. diff --git a/issue_template.md b/issue_template.md index 2ea213a5..3dbac2ac 100644 --- a/issue_template.md +++ b/issue_template.md @@ -14,4 +14,4 @@ <!-- Paste the output of "mitmproxy --version" here. --> -<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) --> +<!-- Please use StackOverflow (https://stackoverflow.com/questions/tagged/mitmproxy) for support/how-to questions. Thanks! :) --> diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index c56c0e74..7bdaeb33 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -203,8 +203,9 @@ class ClientPlayback: # https://github.com/mitmproxy/mitmproxy/issues/2197 if hf.request.http_version == "HTTP/2.0": hf.request.http_version = "HTTP/1.1" - host = hf.request.headers.pop(":authority") - hf.request.headers.insert(0, "host", host) + host = hf.request.headers.pop(":authority", None) + if host is not None: + hf.request.headers.insert(0, "host", host) self.q.put(hf) ctx.master.addons.trigger("update", lst) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 90e95d3e..2776118a 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,17 +11,23 @@ import mitmproxy.types import pyperclip -def raise_if_missing_request(f: flow.Flow) -> None: +def cleanup_request(f: flow.Flow): if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") + request = f.request.copy() # type: ignore + request.decode(strict=False) + # a bit of clean-up + if request.method == 'GET' and request.headers.get("content-length", None) == "0": + request.headers.pop('content-length') + request.headers.pop(':authority', None) + return request def curl_command(f: flow.Flow) -> str: - raise_if_missing_request(f) data = "curl " - request = f.request.copy() # type: ignore - request.decode(strict=False) + request = cleanup_request(f) for k, v in request.headers.items(multi=True): + data += "--compressed " if k == 'accept-encoding' else "" data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method @@ -35,11 +41,8 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: - raise_if_missing_request(f) - request = f.request.copy() # type: ignore - data = "http %s " % request.method - request.decode(strict=False) - data += "%s" % request.url + request = cleanup_request(f) + data = "http %s %s" % (request.method, request.url) for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) if request.content: @@ -51,8 +54,7 @@ def httpie_command(f: flow.Flow) -> str: def raw(f: flow.Flow) -> bytes: - raise_if_missing_request(f) - return assemble.assemble_request(f.request) # type: ignore + return assemble.assemble_request(cleanup_request(f)) # type: ignore formats = dict( diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 6830c6cd..d5a7ff9c 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,14 +1,18 @@ import re -from typing import Optional, Union # noqa +from typing import Optional # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding from mitmproxy.coretypes import serializable -from mitmproxy.net.http import headers +from mitmproxy.net.http import headers as mheaders class MessageData(serializable.Serializable): - content: bytes = None + headers: mheaders.Headers + content: bytes + http_version: bytes + timestamp_start: float + timestamp_end: float def __eq__(self, other): if isinstance(other, MessageData): @@ -18,7 +22,7 @@ class MessageData(serializable.Serializable): def set_state(self, state): for k, v in state.items(): if k == "headers": - v = headers.Headers.from_state(v) + v = mheaders.Headers.from_state(v) setattr(self, k, v) def get_state(self): @@ -28,12 +32,12 @@ class MessageData(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) class Message(serializable.Serializable): - data: MessageData = None + data: MessageData def __eq__(self, other): if isinstance(other, Message): @@ -48,7 +52,7 @@ class Message(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) @property @@ -160,7 +164,7 @@ class Message(serializable.Serializable): self.data.timestamp_end = timestamp_end def _get_content_type_charset(self) -> Optional[str]: - ct = headers.parse_content_type(self.headers.get("content-type", "")) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") return None @@ -213,9 +217,9 @@ class Message(serializable.Serializable): self.content = encoding.encode(text, enc) except ValueError: # Fall back to UTF-8 and update the content-type header. - ct = headers.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) ct[2]["charset"] = "utf-8" - self.headers["content-type"] = headers.assemble_content_type(*ct) + self.headers["content-type"] = mheaders.assemble_content_type(*ct) enc = "utf8" self.content = text.encode(enc, "surrogateescape") diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index ef33ca49..1569ea72 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -64,6 +64,8 @@ class Request(message.Message): """ An HTTP request. """ + data: RequestData + def __init__(self, *args, **kwargs): super().__init__() self.data = RequestData(*args, **kwargs) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 9491fc03..2e864405 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -47,6 +47,8 @@ class Response(message.Message): """ An HTTP response. """ + data: ResponseData + def __init__(self, *args, **kwargs): super().__init__() self.data = ResponseData(*args, **kwargs) diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index 5e22ec31..74e077a4 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -13,9 +13,15 @@ def lookup(address, port, s): # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() - spec = "%s:%s" % (address, port) + + # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443 ESTABLISHED:ESTABLISHED + specv4 = "%s:%s" % (address, port) + + # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443] ESTABLISHED:ESTABLISHED + specv6 = "%s[%s]" % (address, port) + for i in s.split("\n"): - if "ESTABLISHED:ESTABLISHED" in i and spec in i: + if "ESTABLISHED:ESTABLISHED" in i and specv4 in i: s = i.split() if len(s) > 4: if sys.platform.startswith("freebsd"): @@ -26,4 +32,11 @@ def lookup(address, port, s): if len(s) == 2: return s[0], int(s[1]) + elif "ESTABLISHED:ESTABLISHED" in i and specv6 in i: + s = i.split() + if len(s) > 4: + s = s[4].split("[") + port = s[1].split("]") + port = port[0] + return s[0], int(port) raise RuntimeError("Could not resolve original destination.") diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index 5c02b072..3158a294 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -48,12 +48,14 @@ def parse_size(s: typing.Optional[str]) -> typing.Optional[int]: raise ValueError("Invalid size specification.") -def pretty_duration(secs): +def pretty_duration(secs: typing.Optional[float]) -> str: formatters = [ (100, "{:.0f}s"), (10, "{:2.1f}s"), (1, "{:1.2f}s"), ] + if secs is None: + return "" for limit, formatter in formatters: if secs >= limit: diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index f4bb0f64..c86e0c7d 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -14,29 +14,23 @@ from unittest import mock @pytest.fixture def get_request(): return tflow.tflow( - req=tutils.treq( - method=b'GET', - content=b'', - path=b"/path?a=foo&a=bar&b=baz" - ) - ) + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz")) @pytest.fixture def post_request(): return tflow.tflow( - req=tutils.treq( - method=b'POST', - headers=(), - content=bytes(range(256)) - ) - ) + req=tutils.treq(method=b'POST', headers=(), content=bytes(range(256)))) @pytest.fixture def patch_request(): return tflow.tflow( - req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + req=tutils.treq( + method=b'PATCH', + content=b'content', + path=b"/path?query=param" + ) ) @@ -47,7 +41,7 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' -H 'content-length:0' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header:qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): @@ -67,7 +61,7 @@ class TestExportCurlCommand: class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue' 'content-length:0'""" + result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): diff --git a/test/mitmproxy/data/pf01 b/test/mitmproxy/data/pf01 index 3139a289..019a6b76 100644 --- a/test/mitmproxy/data/pf01 +++ b/test/mitmproxy/data/pf01 @@ -1,4 +1,10 @@ No ALTQ support in kernel ALTQ related functions disabled +ALL tcp 192.168.1.111:40001 -> 5.5.5.6:80 FIN_WAIT_2:FIN_WAIT_2 ALL tcp 127.0.0.1:8080 <- 5.5.5.6:80 <- 192.168.1.111:40001 FIN_WAIT_2:FIN_WAIT_2 +ALL tcp 192.168.1.111:40000 -> 5.5.5.5:80 ESTABLISHED:ESTABLISHED ALL tcp 127.0.0.1:8080 <- 5.5.5.5:80 <- 192.168.1.111:40000 ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] -> 2a03:2880:f21f:c5:face:b00c::167[443] ESTABLISHED:ESTABLISHED +ALL tcp ::1[8080] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] -> 2a03:2880:f21f:c5:face:b00c::167[443] FIN_WAIT_2:FIN_WAIT_2 +ALL tcp ::1[6970] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] FIN_WAIT_2:FIN_WAIT_2
\ No newline at end of file diff --git a/test/mitmproxy/platform/test_pf.py b/test/mitmproxy/platform/test_pf.py index 9795a2db..4a7dfe75 100644 --- a/test/mitmproxy/platform/test_pf.py +++ b/test/mitmproxy/platform/test_pf.py @@ -19,3 +19,8 @@ class TestLookup: pf.lookup("192.168.1.112", 40000, d) with pytest.raises(Exception, match="Could not resolve original destination"): pf.lookup("192.168.1.111", 40001, d) + assert pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40002, d) == ("2a03:2880:f21f:c5:face:b00c::167", 443) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40003, d) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:face:face:face:face:face:face", 40003, d) diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index faf35f72..6f8bf732 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -47,6 +47,7 @@ def test_pretty_duration(): assert human.pretty_duration(10000) == "10000s" assert human.pretty_duration(1.123) == "1.12s" assert human.pretty_duration(0.123) == "123ms" + assert human.pretty_duration(None) == "" def test_format_address(): |