diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | examples/complex/dns_spoofing.py | 4 | ||||
| -rw-r--r-- | examples/simple/custom_contentview.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/__init__.py | 5 | ||||
| -rw-r--r-- | mitmproxy/contentviews/protobuf.py | 21 | ||||
| -rw-r--r-- | mitmproxy/export.py | 4 | ||||
| -rw-r--r-- | mitmproxy/net/http/http1/assemble.py | 3 | ||||
| -rw-r--r-- | mitmproxy/net/http/http1/read.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/request.py | 47 | ||||
| -rw-r--r-- | mitmproxy/options.py | 15 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/http.py | 7 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/http2.py | 15 | ||||
| -rw-r--r-- | mitmproxy/tools/cmdline.py | 27 | ||||
| -rw-r--r-- | setup.cfg | 63 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rw-r--r-- | test/full_coverage_plugin.py | 4 | ||||
| -rw-r--r-- | test/individual_coverage.py | 82 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf.py | 52 | ||||
| -rw-r--r-- | test/mitmproxy/net/http/test_request.py | 47 | ||||
| -rw-r--r-- | test/mitmproxy/proxy/protocol/test_http2.py | 142 | ||||
| -rw-r--r-- | test/mitmproxy/test_flow.py | 112 | ||||
| -rw-r--r-- | test/mitmproxy/test_tcp.py | 60 | ||||
| -rw-r--r-- | test/mitmproxy/test_websocket.py | 63 | ||||
| -rw-r--r-- | tox.ini | 6 |
24 files changed, 531 insertions, 257 deletions
diff --git a/.travis.yml b/.travis.yml index f534100b..4c85c46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,8 @@ matrix: packages: - libssl-dev - python: 3.5 + env: TOXENV=individual_coverage + - python: 3.5 env: TOXENV=docs install: diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index c6d46b66..acda303d 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -34,7 +34,7 @@ class Rerouter: The original host header is retrieved early before flow.request is replaced by mitmproxy new outgoing request """ - flow.metadata["original_host"] = flow.request.headers["Host"] + flow.metadata["original_host"] = flow.request.host_header def request(self, flow): if flow.client_conn.ssl_established: @@ -53,7 +53,7 @@ class Rerouter: if m.group("port"): port = int(m.group("port")) - flow.request.headers["Host"] = host_header + flow.request.host_header = host_header flow.request.host = sni or host_header flow.request.port = port diff --git a/examples/simple/custom_contentview.py b/examples/simple/custom_contentview.py index 35216397..1f3a38ec 100644 --- a/examples/simple/custom_contentview.py +++ b/examples/simple/custom_contentview.py @@ -10,7 +10,7 @@ class ViewSwapCase(contentviews.View): # We don't have a good solution for the keyboard shortcut yet - # you manually need to find a free letter. Contributions welcome :) - prompt = ("swap case text", "p") + prompt = ("swap case text", "z") content_types = ["text/plain"] def __call__(self, data: bytes, **metadata): diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 357172e3..c7db6690 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -159,6 +159,7 @@ def get_content_view(viewmode: View, data: bytes, **metadata): return desc, safe_to_print(content), error +# The order in which ContentViews are added is important! add(auto.ViewAuto()) add(raw.ViewRaw()) add(hex.ViewHex()) @@ -172,9 +173,7 @@ add(urlencoded.ViewURLEncoded()) add(multipart.ViewMultipart()) add(image.ViewImage()) add(query.ViewQuery()) - -if protobuf.ViewProtobuf.is_available(): - add(protobuf.ViewProtobuf()) +add(protobuf.ViewProtobuf()) __all__ = [ "View", "VIEW_CUTOFF", "KEY_MAX", "format_text", "format_dict", diff --git a/mitmproxy/contentviews/protobuf.py b/mitmproxy/contentviews/protobuf.py index 620d9444..4bbb1580 100644 --- a/mitmproxy/contentviews/protobuf.py +++ b/mitmproxy/contentviews/protobuf.py @@ -15,31 +15,28 @@ class ViewProtobuf(base.View): "application/x-protobuffer", ] - @staticmethod - def is_available(): + def is_available(self): try: p = subprocess.Popen( ["protoc", "--version"], stdout=subprocess.PIPE ) out, _ = p.communicate() - return out.startswith("libprotoc") + return out.startswith(b"libprotoc") except: return False - def decode_protobuf(self, content): + def __call__(self, data, **metadata): + if not self.is_available(): + raise NotImplementedError("protoc not found. Please make sure 'protoc' is available in $PATH.") + # if Popen raises OSError, it will be caught in # get_content_view and fall back to Raw p = subprocess.Popen(['protoc', '--decode_raw'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate(input=content) - if out: - return out - else: - return err - - def __call__(self, data, **metadata): - decoded = self.decode_protobuf(data) + decoded, _ = p.communicate(input=data) + if not decoded: + raise ValueError("Failed to parse input.") return "Protobuf", base.format_text(decoded) diff --git a/mitmproxy/export.py b/mitmproxy/export.py index 0a261509..235e754a 100644 --- a/mitmproxy/export.py +++ b/mitmproxy/export.py @@ -73,7 +73,7 @@ def python_code(flow: http.HTTPFlow): headers = flow.request.headers.copy() # requests adds those by default. - for x in ("host", "content-length"): + for x in (":authority", "host", "content-length"): headers.pop(x, None) writearg("headers", dict(headers)) try: @@ -130,7 +130,7 @@ def locust_code(flow): if flow.request.headers: lines = [ (_native(k), _native(v)) for k, v in flow.request.headers.fields - if _native(k).lower() not in ["host", "cookie"] + if _native(k).lower() not in [":authority", "host", "cookie"] ] lines = [" '%s': '%s',\n" % (k, v) for k, v in lines] headers += "\n headers = {\n%s }\n" % "".join(lines) diff --git a/mitmproxy/net/http/http1/assemble.py b/mitmproxy/net/http/http1/assemble.py index d718589f..8b7246f7 100644 --- a/mitmproxy/net/http/http1/assemble.py +++ b/mitmproxy/net/http/http1/assemble.py @@ -78,8 +78,9 @@ def _assemble_request_headers(request_data): Args: request_data (mitmproxy.net.http.request.RequestData) """ - headers = request_data.headers.copy() + headers = request_data.headers if "host" not in headers and request_data.scheme and request_data.host and request_data.port: + headers = headers.copy() headers["host"] = mitmproxy.net.http.url.hostport( request_data.scheme, request_data.host, diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index d0493da4..ef88fd6c 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -227,7 +227,7 @@ def _get_first_line(rfile): if line == b"\r\n" or line == b"\n": # Possible leftover from previous message line = rfile.readline() - except exceptions.TcpDisconnect: + except (exceptions.TcpDisconnect, exceptions.TlsException): raise exceptions.HttpReadDisconnect("Remote disconnected") if not line: raise exceptions.HttpReadDisconnect("Remote disconnected") diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 68a11ce7..b961e1e4 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +from typing import Optional from mitmproxy.types import multidict from mitmproxy.utils import strutils @@ -164,11 +165,44 @@ class Request(message.Message): self.data.host = host # Update host header - if "host" in self.headers: - if host: - self.headers["host"] = host + if self.host_header is not None: + self.host_header = host + + @property + def host_header(self) -> Optional[str]: + """ + The request's host/authority header. + + This property maps to either ``request.headers["Host"]`` or + ``request.headers[":authority"]``, depending on whether it's HTTP/1.x or HTTP/2.0. + """ + if ":authority" in self.headers: + return self.headers[":authority"] + if "Host" in self.headers: + return self.headers["Host"] + return None + + @host_header.setter + def host_header(self, val: Optional[str]) -> None: + if val is None: + self.headers.pop("Host", None) + self.headers.pop(":authority", None) + elif self.host_header is not None: + # Update any existing headers. + if ":authority" in self.headers: + self.headers[":authority"] = val + if "Host" in self.headers: + self.headers["Host"] = val + else: + # Only add the correct new header. + if self.http_version.upper().startswith("HTTP/2"): + self.headers[":authority"] = val else: - self.headers.pop("host") + self.headers["Host"] = val + + @host_header.deleter + def host_header(self): + self.host_header = None @property def port(self): @@ -211,9 +245,10 @@ class Request(message.Message): def _parse_host_header(self): """Extract the host and port from Host header""" - if "host" not in self.headers: + host = self.host_header + if not host: return None, None - host, port = self.headers["host"], None + port = None m = host_header_re.match(host) if m: host = m.group("host").strip("[]") diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 630d2964..2467b9dd 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -54,6 +54,7 @@ class Options(optmanager.OptManager): server_replay_ignore_params: Sequence[str] = [], server_replay_ignore_payload_params: Sequence[str] = [], server_replay_ignore_host: bool = False, + # Proxy options auth_nonanonymous: bool = False, auth_singleuser: Optional[str] = None, @@ -65,15 +66,18 @@ class Options(optmanager.OptManager): ciphers_client: str=DEFAULT_CLIENT_CIPHERS, ciphers_server: Optional[str]=None, clientcerts: Optional[str] = None, - http2: bool = True, ignore_hosts: Sequence[str] = [], listen_host: str = "", listen_port: int = LISTEN_PORT, upstream_bind_address: str = "", mode: str = "regular", no_upstream_cert: bool = False, - rawtcp: bool = False, + + http2: bool = True, + http2_priority: bool = False, websocket: bool = True, + rawtcp: bool = False, + spoof_source_address: bool = False, upstream_server: Optional[str] = None, upstream_auth: Optional[str] = None, @@ -152,15 +156,18 @@ class Options(optmanager.OptManager): self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts - self.http2 = http2 self.ignore_hosts = ignore_hosts self.listen_host = listen_host self.listen_port = listen_port self.upstream_bind_address = upstream_bind_address self.mode = mode self.no_upstream_cert = no_upstream_cert - self.rawtcp = rawtcp + + self.http2 = http2 + self.http2_priority = http2_priority self.websocket = websocket + self.rawtcp = rawtcp + self.spoof_source_address = spoof_source_address self.upstream_server = upstream_server self.upstream_auth = upstream_auth diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index da9a8781..a7d56f24 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -291,7 +291,7 @@ class HttpLayer(base.Layer): # update host header in reverse proxy mode if self.config.options.mode == "reverse": - f.request.headers["Host"] = self.config.upstream_server.address.host + f.request.host_header = self.config.upstream_server.address.host # Determine .scheme, .host and .port attributes for inline scripts. For # absolute-form requests, they are directly given in the request. For @@ -301,11 +301,10 @@ class HttpLayer(base.Layer): if self.mode is HTTPMode.transparent: # Setting request.host also updates the host header, which we want # to preserve - host_header = f.request.headers.get("host", None) + host_header = f.request.host_header f.request.host = self.__initial_server_conn.address.host f.request.port = self.__initial_server_conn.address.port - if host_header: - f.request.headers["host"] = host_header + f.request.host_header = host_header # set again as .host overwrites this. f.request.scheme = "https" if self.__initial_server_tls else "http" self.channel.ask("request", f) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index b7548221..cdce24b3 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -268,6 +268,10 @@ class Http2Layer(base.Layer): return True def _handle_priority_updated(self, eid, event): + if not self.config.options.http2_priority: + self.log("HTTP/2 PRIORITY frame surpressed. Use --http2-priority to enable forwarding.", "debug") + return True + if eid in self.streams and self.streams[eid].handled_priority_event is event: # this event was already handled during stream creation # HeadersFrame + Priority information as RequestReceived @@ -527,9 +531,12 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr if self.handled_priority_event: # only send priority information if they actually came with the original HeadersFrame # and not if they got updated before/after with a PriorityFrame - priority_exclusive = self.priority_exclusive - priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) - priority_weight = self.priority_weight + if not self.config.options.http2_priority: + self.log("HTTP/2 PRIORITY information in HEADERS frame surpressed. Use --http2-priority to enable forwarding.", "debug") + else: + priority_exclusive = self.priority_exclusive + priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) + priority_weight = self.priority_weight try: self.connections[self.server_conn].safe_send_headers( @@ -610,7 +617,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr chunks ) - def __call__(self): + def __call__(self): # pragma: no cover raise EnvironmentError('Http2SingleStreamLayer must be run as thread') def run(self): diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 1c620fd6..bb0bb17a 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -137,7 +137,6 @@ def get_common_options(args): ciphers_client = args.ciphers_client, ciphers_server = args.ciphers_server, clientcerts = args.clientcerts, - http2 = args.http2, ignore_hosts = args.ignore_hosts, listen_host = args.addr, listen_port = args.port, @@ -145,8 +144,12 @@ def get_common_options(args): mode = mode, no_upstream_cert = args.no_upstream_cert, spoof_source_address = args.spoof_source_address, - rawtcp = args.rawtcp, + + http2 = args.http2, + http2_priority = args.http2_priority, websocket = args.websocket, + rawtcp = args.rawtcp, + upstream_server = upstream_server, upstream_auth = args.upstream_auth, ssl_version_client = args.ssl_version_client, @@ -334,18 +337,26 @@ def proxy_options(parser): ) http2 = group.add_mutually_exclusive_group() - http2.add_argument("--no-http2", action="store_false", dest="http2", + http2.add_argument("--no-http2", action="store_false", dest="http2") + http2.add_argument("--http2", action="store_true", dest="http2", help="Explicitly enable/disable HTTP/2 support. " - "Enabled by default." + "HTTP/2 support is enabled by default.", ) - http2.add_argument("--http2", action="store_true", dest="http2") + + http2_priority = group.add_mutually_exclusive_group() + http2_priority.add_argument("--http2-priority", action="store_true", dest="http2_priority") + http2_priority.add_argument("--no-http2-priority", action="store_false", dest="http2_priority", + help="Explicitly enable/disable PRIORITY forwarding for HTTP/2 connections. " + "PRIORITY forwarding is disabled by default, " + "because some webservers fail at implementing the RFC properly.", + ) websocket = group.add_mutually_exclusive_group() - websocket.add_argument("--no-websocket", action="store_false", dest="websocket", + websocket.add_argument("--no-websocket", action="store_false", dest="websocket") + websocket.add_argument("--websocket", action="store_true", dest="websocket", help="Explicitly enable/disable WebSocket support. " - "Enabled by default." + "WebSocket support is enabled by default.", ) - websocket.add_argument("--websocket", action="store_true", dest="websocket") parser.add_argument( "--upstream-auth", @@ -22,7 +22,6 @@ exclude_lines = [tool:full_coverage] exclude = mitmproxy/contentviews/__init__.py - mitmproxy/contentviews/protobuf.py mitmproxy/contentviews/wbxml.py mitmproxy/contentviews/xml_html.py mitmproxy/net/tcp.py @@ -49,3 +48,65 @@ exclude = pathod/pathod.py pathod/test.py pathod/protocols/http2.py + +[tool:individual_coverage] +exclude = + mitmproxy/addonmanager.py + mitmproxy/addons/onboardingapp/app.py + mitmproxy/addons/termlog.py + mitmproxy/certs.py + mitmproxy/connections.py + mitmproxy/contentviews/base.py + mitmproxy/contentviews/wbxml.py + mitmproxy/contentviews/xml_html.py + mitmproxy/controller.py + mitmproxy/ctx.py + mitmproxy/exceptions.py + mitmproxy/export.py + mitmproxy/flow.py + mitmproxy/flowfilter.py + mitmproxy/http.py + mitmproxy/io.py + mitmproxy/io_compat.py + mitmproxy/log.py + mitmproxy/master.py + mitmproxy/net/check.py + mitmproxy/net/http/cookies.py + mitmproxy/net/http/headers.py + mitmproxy/net/http/message.py + mitmproxy/net/http/multipart.py + mitmproxy/net/http/url.py + mitmproxy/net/tcp.py + mitmproxy/options.py + mitmproxy/optmanager.py + mitmproxy/proxy/config.py + mitmproxy/proxy/modes/http_proxy.py + mitmproxy/proxy/modes/reverse_proxy.py + mitmproxy/proxy/modes/socks_proxy.py + mitmproxy/proxy/modes/transparent_proxy.py + mitmproxy/proxy/protocol/base.py + mitmproxy/proxy/protocol/http.py + mitmproxy/proxy/protocol/http1.py + mitmproxy/proxy/protocol/http2.py + mitmproxy/proxy/protocol/http_replay.py + mitmproxy/proxy/protocol/rawtcp.py + mitmproxy/proxy/protocol/tls.py + mitmproxy/proxy/protocol/websocket.py + mitmproxy/proxy/root_context.py + mitmproxy/proxy/server.py + mitmproxy/stateobject.py + mitmproxy/types/multidict.py + mitmproxy/utils/bits.py + pathod/language/actions.py + pathod/language/base.py + pathod/language/exceptions.py + pathod/language/generators.py + pathod/language/http.py + pathod/language/message.py + pathod/log.py + pathod/pathoc.py + pathod/pathod.py + pathod/protocols/http.py + pathod/protocols/http2.py + pathod/protocols/websockets.py + pathod/test.py @@ -110,9 +110,6 @@ setup( "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ - "protobuf>=3.1.0, <3.3", - # TODO: Find Python 3 replacement - # "pyamf>=0.8.0, <0.9", ], 'examples': [ "beautifulsoup4>=4.4.1, <4.6", diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py index e9951af9..d98c29d6 100644 --- a/test/full_coverage_plugin.py +++ b/test/full_coverage_plugin.py @@ -2,6 +2,8 @@ import os import configparser import pytest +here = os.path.abspath(os.path.dirname(__file__)) + enable_coverage = False coverage_values = [] @@ -36,7 +38,7 @@ def pytest_configure(config): ) c = configparser.ConfigParser() - c.read('setup.cfg') + c.read(os.path.join(here, "..", "setup.cfg")) fs = c['tool:full_coverage']['exclude'].split('\n') no_full_cov = config.option.no_full_cov + [f.strip() for f in fs] diff --git a/test/individual_coverage.py b/test/individual_coverage.py new file mode 100644 index 00000000..35bcd27f --- /dev/null +++ b/test/individual_coverage.py @@ -0,0 +1,82 @@ +import io +import contextlib +import os +import sys +import glob +import multiprocessing +import configparser +import itertools +import pytest + + +def run_tests(src, test, fail): + stderr = io.StringIO() + stdout = io.StringIO() + with contextlib.redirect_stderr(stderr): + with contextlib.redirect_stdout(stdout): + e = pytest.main([ + '-qq', + '--disable-pytest-warnings', + '--no-faulthandler', + '--cov', src.replace('.py', '').replace('/', '.'), + '--cov-fail-under', '100', + '--cov-report', 'term-missing:skip-covered', + test + ]) + + if e == 0: + if fail: + print("SUCCESS but should have FAILED:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") + e = 42 + else: + print("SUCCESS:", src) + else: + if fail: + print("Ignoring fail:", src) + e = 0 + else: + cov = [l for l in stdout.getvalue().split("\n") if (src in l) or ("was never imported" in l)] + if len(cov) == 1: + print("FAIL:", cov[0]) + else: + print("FAIL:", src, test, stdout.getvalue(), stdout.getvalue()) + print(stderr.getvalue()) + print(stdout.getvalue()) + + sys.exit(e) + + +def start_pytest(src, test, fail): + # run pytest in a new process, otherwise imports and modules might conflict + proc = multiprocessing.Process(target=run_tests, args=(src, test, fail)) + proc.start() + proc.join() + return (src, test, proc.exitcode) + + +def main(): + c = configparser.ConfigParser() + c.read('setup.cfg') + fs = c['tool:individual_coverage']['exclude'].strip().split('\n') + no_individual_cov = [f.strip() for f in fs] + + excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] + src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) + src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] + src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] + + ps = [] + for src in sorted(src_files): + test = os.path.join("test", os.path.dirname(src), "test_" + os.path.basename(src)) + if os.path.isfile(test): + ps.append((src, test, src in no_individual_cov)) + + result = list(itertools.starmap(start_pytest, ps)) + + if any(e != 0 for _, _, e in result): + sys.exit(1) + pass + + +if __name__ == '__main__': + main() diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index 1224b8db..31e382ec 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -1,12 +1,50 @@ +from unittest import mock +import pytest + from mitmproxy.contentviews import protobuf from mitmproxy.test import tutils from . import full_eval -if protobuf.ViewProtobuf.is_available(): - def test_view_protobuf_request(): - v = full_eval(protobuf.ViewProtobuf()) - p = tutils.test_data.path("mitmproxy/data/protobuf01") - content_type, output = v(open(p, "rb").read()) - assert content_type == "Protobuf" - assert output.next()[0][1] == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"' +def test_view_protobuf_request(): + v = full_eval(protobuf.ViewProtobuf()) + p = tutils.test_data.path("mitmproxy/data/protobuf01") + + with mock.patch('mitmproxy.contentviews.protobuf.ViewProtobuf.is_available'): + with mock.patch('subprocess.Popen') as n: + m = mock.Mock() + attrs = {'communicate.return_value': (b'1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"', True)} + m.configure_mock(**attrs) + n.return_value = m + + content_type, output = v(open(p, "rb").read()) + assert content_type == "Protobuf" + assert output[0] == [('text', b'1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"')] + + m.communicate = mock.MagicMock() + m.communicate.return_value = (None, None) + with pytest.raises(ValueError, matches="Failed to parse input."): + v(b'foobar') + + +def test_view_protobuf_availability(): + with mock.patch('subprocess.Popen') as n: + m = mock.Mock() + attrs = {'communicate.return_value': (b'libprotoc fake version', True)} + m.configure_mock(**attrs) + n.return_value = m + assert protobuf.ViewProtobuf().is_available() + + m = mock.Mock() + attrs = {'communicate.return_value': (b'command not found', True)} + m.configure_mock(**attrs) + n.return_value = m + assert not protobuf.ViewProtobuf().is_available() + + +def test_view_protobuf_fallback(): + with mock.patch('subprocess.Popen.communicate') as m: + m.side_effect = OSError() + v = full_eval(protobuf.ViewProtobuf()) + with pytest.raises(NotImplementedError, matches='protoc not found'): + v(b'foobar') diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 6fe57010..90ec31fe 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -97,7 +97,7 @@ class TestRequestCore: request.host = d assert request.data.host == b"foo\xFF\x00bar" - def test_host_header_update(self): + def test_host_update_also_updates_header(self): request = treq() assert "host" not in request.headers request.host = "example.com" @@ -107,6 +107,51 @@ class TestRequestCore: request.host = "example.org" assert request.headers["Host"] == "example.org" + def test_get_host_header(self): + no_hdr = treq() + assert no_hdr.host_header is None + + h1 = treq(headers=( + (b"host", b"example.com"), + )) + assert h1.host_header == "example.com" + + h2 = treq(headers=( + (b":authority", b"example.org"), + )) + assert h2.host_header == "example.org" + + both_hdrs = treq(headers=( + (b"host", b"example.org"), + (b":authority", b"example.com"), + )) + assert both_hdrs.host_header == "example.com" + + def test_modify_host_header(self): + h1 = treq() + assert "host" not in h1.headers + assert ":authority" not in h1.headers + h1.host_header = "example.com" + assert "host" in h1.headers + assert ":authority" not in h1.headers + h1.host_header = None + assert "host" not in h1.headers + + h2 = treq(http_version=b"HTTP/2.0") + h2.host_header = "example.org" + assert "host" not in h2.headers + assert ":authority" in h2.headers + del h2.host_header + assert ":authority" not in h2.headers + + both_hdrs = treq(headers=( + (b":authority", b"example.com"), + (b"host", b"example.org"), + )) + both_hdrs.host_header = "foo.example.com" + assert both_hdrs.headers["Host"] == "foo.example.com" + assert both_hdrs.headers[":authority"] == "foo.example.com" + class TestRequestUtils: """ diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index cede0b80..cb9c0474 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -4,7 +4,7 @@ import os import tempfile import traceback - +import pytest import h2 from mitmproxy import options @@ -272,13 +272,13 @@ class TestSimple(_Http2Test): @requires_alpn -class TestRequestWithPriority(_Http2Test): +class TestForbiddenHeaders(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - elif isinstance(event, h2.events.RequestReceived): + elif isinstance(event, h2.events.StreamEnded): import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -289,60 +289,18 @@ class TestRequestWithPriority(_Http2Test): warnings.simplefilter("ignore") - headers = [(':status', '200')] - if event.priority_updated: - headers.append(('priority_exclusive', str(event.priority_updated.exclusive).encode())) - headers.append(('priority_depends_on', str(event.priority_updated.depends_on).encode())) - headers.append(('priority_weight', str(event.priority_updated.weight).encode())) - h2_conn.send_headers(event.stream_id, headers) + h2_conn.config.validate_outbound_headers = False + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('keep-alive', 'foobar'), + ]) + h2_conn.send_data(event.stream_id, b'response body') h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_request_with_priority(self): - client, h2_conn = self._setup_connection() - - self._send_request( - client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address.port)), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - priority_exclusive=True, - priority_depends_on=42424242, - priority_weight=42, - ) - - done = False - while not done: - try: - raw = b''.join(http2.read_raw_frame(client.rfile)) - events = h2_conn.receive_data(raw) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response.headers['priority_exclusive'] == 'True' - assert self.master.state.flows[0].response.headers['priority_depends_on'] == '42424242' - assert self.master.state.flows[0].response.headers['priority_weight'] == '42' - - def test_request_without_priority(self): + def test_forbidden_headers(self): client, h2_conn = self._setup_connection() self._send_request( @@ -353,8 +311,7 @@ class TestRequestWithPriority(_Http2Test): (':method', 'GET'), (':scheme', 'https'), (':path', '/'), - ], - ) + ]) done = False while not done: @@ -369,7 +326,9 @@ class TestRequestWithPriority(_Http2Test): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.ResponseReceived): + assert 'keep-alive' not in event.headers + elif isinstance(event, h2.events.StreamEnded): done = True h2_conn.close_connection() @@ -377,21 +336,17 @@ class TestRequestWithPriority(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert 'priority_exclusive' not in self.master.state.flows[0].response.headers - assert 'priority_depends_on' not in self.master.state.flows[0].response.headers - assert 'priority_weight' not in self.master.state.flows[0].response.headers + assert self.master.state.flows[0].response.status_code == 200 + assert self.master.state.flows[0].response.headers['keep-alive'] == 'foobar' @requires_alpn -class TestPriority(_Http2Test): - priority_data = None +class TestRequestWithPriority(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - elif isinstance(event, h2.events.PriorityUpdated): - cls.priority_data = (event.exclusive, event.depends_on, event.weight) elif isinstance(event, h2.events.RequestReceived): import warnings with warnings.catch_warnings(): @@ -404,18 +359,26 @@ class TestPriority(_Http2Test): warnings.simplefilter("ignore") headers = [(':status', '200')] + if event.priority_updated: + headers.append(('priority_exclusive', str(event.priority_updated.exclusive).encode())) + headers.append(('priority_depends_on', str(event.priority_updated.depends_on).encode())) + headers.append(('priority_weight', str(event.priority_updated.weight).encode())) h2_conn.send_headers(event.stream_id, headers) h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_priority(self): - client, h2_conn = self._setup_connection() + @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ + (True, (True, 42424242, 42), ('True', '42424242', '42')), + (False, (True, 42424242, 42), (None, None, None)), + (True, (None, None, None), (None, None, None)), + (False, (None, None, None), (None, None, None)), + ]) + def test_request_with_priority(self, http2_priority_enabled, priority, expected_priority): + self.config.options.http2_priority = http2_priority_enabled - h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() + client, h2_conn = self._setup_connection() self._send_request( client.wfile, @@ -426,6 +389,9 @@ class TestPriority(_Http2Test): (':scheme', 'https'), (':path', '/'), ], + priority_exclusive=priority[0], + priority_depends_on=priority[1], + priority_weight=priority[2], ) done = False @@ -449,12 +415,15 @@ class TestPriority(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert self.priority_data == (True, 0, 42) + + resp = self.master.state.flows[0].response + assert resp.headers.get('priority_exclusive', None) == expected_priority[0] + assert resp.headers.get('priority_depends_on', None) == expected_priority[1] + assert resp.headers.get('priority_weight', None) == expected_priority[2] @requires_alpn -class TestPriorityWithExistingStream(_Http2Test): - priority_data = [] +class TestPriority(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): @@ -463,8 +432,6 @@ class TestPriorityWithExistingStream(_Http2Test): elif isinstance(event, h2.events.PriorityUpdated): cls.priority_data.append((event.exclusive, event.depends_on, event.weight)) elif isinstance(event, h2.events.RequestReceived): - assert not event.priority_updated - import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -477,17 +444,27 @@ class TestPriorityWithExistingStream(_Http2Test): headers = [(':status', '200')] h2_conn.send_headers(event.stream_id, headers) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - elif isinstance(event, h2.events.StreamEnded): h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_priority_with_existing_stream(self): + @pytest.mark.parametrize("prioritize_before", [True, False]) + @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ + (True, (True, 42424242, 42), [(True, 42424242, 42)]), + (False, (True, 42424242, 42), []), + ]) + def test_priority(self, prioritize_before, http2_priority_enabled, priority, expected_priority): + self.config.options.http2_priority = http2_priority_enabled + self.__class__.priority_data = [] + client, h2_conn = self._setup_connection() + if prioritize_before: + h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + self._send_request( client.wfile, h2_conn, @@ -497,13 +474,14 @@ class TestPriorityWithExistingStream(_Http2Test): (':scheme', 'https'), (':path', '/'), ], - end_stream=False, + end_stream=prioritize_before, ) - h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) - h2_conn.end_stream(1) - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() + if not prioritize_before: + h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) + h2_conn.end_stream(1) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() done = False while not done: @@ -526,7 +504,7 @@ class TestPriorityWithExistingStream(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert self.priority_data == [(True, 0, 42)] + assert self.priority_data == expected_priority @requires_alpn diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 65e6845f..a78e5f80 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -10,7 +10,6 @@ from mitmproxy.exceptions import FlowReadException, Kill from mitmproxy import flow from mitmproxy import http from mitmproxy import connections -from mitmproxy import tcp from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer from mitmproxy import master @@ -157,117 +156,6 @@ class TestHTTPFlow: assert f.response.raw_content == b"abarb" -class TestWebSocketFlow: - - def test_copy(self): - f = tflow.twebsocketflow() - f.get_state() - f2 = f.copy() - a = f.get_state() - b = f2.get_state() - del a["id"] - del b["id"] - del a["handshake_flow"]["id"] - del b["handshake_flow"]["id"] - assert a == b - assert not f == f2 - assert f is not f2 - - assert f.client_key == f2.client_key - assert f.client_protocol == f2.client_protocol - assert f.client_extensions == f2.client_extensions - assert f.server_accept == f2.server_accept - assert f.server_protocol == f2.server_protocol - assert f.server_extensions == f2.server_extensions - assert f.messages is not f2.messages - assert f.handshake_flow is not f2.handshake_flow - - for m in f.messages: - m2 = m.copy() - m2.set_state(m2.get_state()) - assert m is not m2 - assert m.get_state() == m2.get_state() - - f = tflow.twebsocketflow(err=True) - f2 = f.copy() - assert f is not f2 - assert f.handshake_flow is not f2.handshake_flow - assert f.error.get_state() == f2.error.get_state() - assert f.error is not f2.error - - def test_match(self): - f = tflow.twebsocketflow() - assert not flowfilter.match("~b nonexistent", f) - assert flowfilter.match(None, f) - assert not flowfilter.match("~b nonexistent", f) - - f = tflow.twebsocketflow(err=True) - assert flowfilter.match("~e", f) - - with pytest.raises(ValueError): - flowfilter.match("~", f) - - def test_repr(self): - f = tflow.twebsocketflow() - assert 'WebSocketFlow' in repr(f) - assert 'binary message: ' in repr(f.messages[0]) - assert 'text message: ' in repr(f.messages[1]) - - -class TestTCPFlow: - - def test_copy(self): - f = tflow.ttcpflow() - f.get_state() - f2 = f.copy() - a = f.get_state() - b = f2.get_state() - del a["id"] - del b["id"] - assert a == b - assert not f == f2 - assert f is not f2 - - assert f.messages is not f2.messages - - for m in f.messages: - assert m.get_state() - m2 = m.copy() - assert not m == m2 - assert m is not m2 - - a = m.get_state() - b = m2.get_state() - assert a == b - - m = tcp.TCPMessage(False, 'foo') - m.set_state(f.messages[0].get_state()) - assert m.timestamp == f.messages[0].timestamp - - f = tflow.ttcpflow(err=True) - f2 = f.copy() - assert f is not f2 - assert f.error.get_state() == f2.error.get_state() - assert f.error is not f2.error - - def test_match(self): - f = tflow.ttcpflow() - assert not flowfilter.match("~b nonexistent", f) - assert flowfilter.match(None, f) - assert not flowfilter.match("~b nonexistent", f) - - f = tflow.ttcpflow(err=True) - assert flowfilter.match("~e", f) - - with pytest.raises(ValueError): - flowfilter.match("~", f) - - def test_repr(self): - f = tflow.ttcpflow() - assert 'TCPFlow' in repr(f) - assert '-> ' in repr(f.messages[0]) - - class TestSerialize: def _treader(self): diff --git a/test/mitmproxy/test_tcp.py b/test/mitmproxy/test_tcp.py index 777ab4dd..dce6493c 100644 --- a/test/mitmproxy/test_tcp.py +++ b/test/mitmproxy/test_tcp.py @@ -1 +1,59 @@ -# TODO: write tests +import pytest + +from mitmproxy import tcp +from mitmproxy import flowfilter +from mitmproxy.test import tflow + + +class TestTCPFlow: + + def test_copy(self): + f = tflow.ttcpflow() + f.get_state() + f2 = f.copy() + a = f.get_state() + b = f2.get_state() + del a["id"] + del b["id"] + assert a == b + assert not f == f2 + assert f is not f2 + + assert f.messages is not f2.messages + + for m in f.messages: + assert m.get_state() + m2 = m.copy() + assert not m == m2 + assert m is not m2 + + a = m.get_state() + b = m2.get_state() + assert a == b + + m = tcp.TCPMessage(False, 'foo') + m.set_state(f.messages[0].get_state()) + assert m.timestamp == f.messages[0].timestamp + + f = tflow.ttcpflow(err=True) + f2 = f.copy() + assert f is not f2 + assert f.error.get_state() == f2.error.get_state() + assert f.error is not f2.error + + def test_match(self): + f = tflow.ttcpflow() + assert not flowfilter.match("~b nonexistent", f) + assert flowfilter.match(None, f) + assert not flowfilter.match("~b nonexistent", f) + + f = tflow.ttcpflow(err=True) + assert flowfilter.match("~e", f) + + with pytest.raises(ValueError): + flowfilter.match("~", f) + + def test_repr(self): + f = tflow.ttcpflow() + assert 'TCPFlow' in repr(f) + assert '-> ' in repr(f.messages[0]) diff --git a/test/mitmproxy/test_websocket.py b/test/mitmproxy/test_websocket.py index 777ab4dd..f2963390 100644 --- a/test/mitmproxy/test_websocket.py +++ b/test/mitmproxy/test_websocket.py @@ -1 +1,62 @@ -# TODO: write tests +import pytest + +from mitmproxy import flowfilter +from mitmproxy.test import tflow + + +class TestWebSocketFlow: + + def test_copy(self): + f = tflow.twebsocketflow() + f.get_state() + f2 = f.copy() + a = f.get_state() + b = f2.get_state() + del a["id"] + del b["id"] + del a["handshake_flow"]["id"] + del b["handshake_flow"]["id"] + assert a == b + assert not f == f2 + assert f is not f2 + + assert f.client_key == f2.client_key + assert f.client_protocol == f2.client_protocol + assert f.client_extensions == f2.client_extensions + assert f.server_accept == f2.server_accept + assert f.server_protocol == f2.server_protocol + assert f.server_extensions == f2.server_extensions + assert f.messages is not f2.messages + assert f.handshake_flow is not f2.handshake_flow + + for m in f.messages: + m2 = m.copy() + m2.set_state(m2.get_state()) + assert m is not m2 + assert m.get_state() == m2.get_state() + + f = tflow.twebsocketflow(err=True) + f2 = f.copy() + assert f is not f2 + assert f.handshake_flow is not f2.handshake_flow + assert f.error.get_state() == f2.error.get_state() + assert f.error is not f2.error + + def test_match(self): + f = tflow.twebsocketflow() + assert not flowfilter.match("~b nonexistent", f) + assert flowfilter.match(None, f) + assert not flowfilter.match("~b nonexistent", f) + + f = tflow.twebsocketflow(err=True) + assert flowfilter.match("~e", f) + + with pytest.raises(ValueError): + flowfilter.match("~", f) + + def test_repr(self): + f = tflow.twebsocketflow() + assert f.message_info(f.messages[0]) + assert 'WebSocketFlow' in repr(f) + assert 'binary message: ' in repr(f.messages[0]) + assert 'text message: ' in repr(f.messages[1]) @@ -36,6 +36,12 @@ commands = mitmproxy/tools/web/ \ mitmproxy/contentviews/ +[testenv:individual_coverage] +deps = + -rrequirements.txt +commands = + python3 test/individual_coverage.py + [testenv:wheel] recreate = True deps = |