diff options
116 files changed, 1074 insertions, 577 deletions
diff --git a/.gitattributes b/.gitattributes index dd08ee53..69d68b8e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ -mitmproxy/tools/web/static/**/* -diff +mitmproxy/tools/web/static/**/* -diff linguist-vendored web/src/js/filt/filt.js -diff diff --git a/.travis.yml b/.travis.yml index e1ff4539..4c85c46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ env: - CI_COMMANDS=codecov git: depth: 10000 + matrix: fast_finish: true include: @@ -43,9 +44,9 @@ matrix: packages: - libssl-dev - python: 3.5 + env: TOXENV=individual_coverage + - python: 3.5 env: TOXENV=docs - allow_failures: - - python: pypy install: - | diff --git a/docs/install.rst b/docs/install.rst index b9524897..cf93cc58 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -85,7 +85,7 @@ libraries. This was tested on a fully patched installation of Ubuntu 16.04. .. code:: bash - sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev + sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev sudo pip3 install mitmproxy # or pip3 install --user mitmproxy On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install @@ -104,7 +104,7 @@ libraries. This was tested on a fully patched installation of Fedora 24. .. code:: bash - sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel + sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel sudo pip3 install mitmproxy # or pip3 install --user mitmproxy Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index 1fb59f74..acda303d 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -1,5 +1,5 @@ """ -This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect +This script makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the Host header of the HTTP request. Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't @@ -34,7 +34,7 @@ class Rerouter: The original host header is retrieved early before flow.request is replaced by mitmproxy new outgoing request """ - flow.metadata["original_host"] = flow.request.headers["Host"] + flow.metadata["original_host"] = flow.request.host_header def request(self, flow): if flow.client_conn.ssl_established: @@ -53,7 +53,7 @@ class Rerouter: if m.group("port"): port = int(m.group("port")) - flow.request.headers["Host"] = host_header + flow.request.host_header = host_header flow.request.host = sni or host_header flow.request.port = port diff --git a/examples/simple/custom_contentview.py b/examples/simple/custom_contentview.py index 35216397..1f3a38ec 100644 --- a/examples/simple/custom_contentview.py +++ b/examples/simple/custom_contentview.py @@ -10,7 +10,7 @@ class ViewSwapCase(contentviews.View): # We don't have a good solution for the keyboard shortcut yet - # you manually need to find a free letter. Contributions welcome :) - prompt = ("swap case text", "p") + prompt = ("swap case text", "z") content_types = ["text/plain"] def __call__(self, data: bytes, **metadata): diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 69d45029..18a85866 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -1,35 +1,43 @@ import binascii +import weakref +from typing import Optional +from typing import Set # noqa +from typing import Tuple import passlib.apache +import mitmproxy.net.http +from mitmproxy import connections # noqa from mitmproxy import exceptions from mitmproxy import http -import mitmproxy.net.http - +from mitmproxy.net.http import status_codes REALM = "mitmproxy" -def mkauth(username, password, scheme="basic"): +def mkauth(username: str, password: str, scheme: str = "basic") -> str: + """ + Craft a basic auth string + """ v = binascii.b2a_base64( (username + ":" + password).encode("utf8") ).decode("ascii") return scheme + " " + v -def parse_http_basic_auth(s): - words = s.split() - if len(words) != 2: - return None - scheme = words[0] +def parse_http_basic_auth(s: str) -> Tuple[str, str, str]: + """ + Parse a basic auth header. + Raises a ValueError if the input is invalid. + """ + scheme, authinfo = s.split() + if scheme.lower() != "basic": + raise ValueError("Unknown scheme") try: - user = binascii.a2b_base64(words[1]).decode("utf8", "replace") - except binascii.Error: - return None - parts = user.split(':') - if len(parts) != 2: - return None - return scheme, parts[0], parts[1] + user, password = binascii.a2b_base64(authinfo.encode()).decode("utf8", "replace").split(":") + except binascii.Error as e: + raise ValueError(str(e)) + return scheme, user, password class ProxyAuth: @@ -37,67 +45,72 @@ class ProxyAuth: self.nonanonymous = False self.htpasswd = None self.singleuser = None + self.mode = None + self.authenticated = weakref.WeakSet() # type: Set[connections.ClientConnection] + """Contains all connections that are permanently authenticated after an HTTP CONNECT""" - def enabled(self): + def enabled(self) -> bool: return any([self.nonanonymous, self.htpasswd, self.singleuser]) - def which_auth_header(self, f): - if f.mode == "regular": + def is_proxy_auth(self) -> bool: + """ + Returns: + - True, if authentication is done as if mitmproxy is a proxy + - False, if authentication is done as if mitmproxy is a HTTP server + """ + return self.mode in ("regular", "upstream") + + def which_auth_header(self) -> str: + if self.is_proxy_auth(): return 'Proxy-Authorization' else: return 'Authorization' - def auth_required_response(self, f): - if f.mode == "regular": - hdrname = 'Proxy-Authenticate' - else: - hdrname = 'WWW-Authenticate' - - headers = mitmproxy.net.http.Headers() - headers[hdrname] = 'Basic realm="%s"' % REALM - - if f.mode == "transparent": + def auth_required_response(self) -> http.HTTPResponse: + if self.is_proxy_auth(): return http.make_error_response( - 401, - "Authentication Required", - headers + status_codes.PROXY_AUTH_REQUIRED, + headers=mitmproxy.net.http.Headers(Proxy_Authenticate='Basic realm="{}"'.format(REALM)), ) else: return http.make_error_response( - 407, - "Proxy Authentication Required", - headers, + status_codes.UNAUTHORIZED, + headers=mitmproxy.net.http.Headers(WWW_Authenticate='Basic realm="{}"'.format(REALM)), ) - def check(self, f): - auth_value = f.request.headers.get(self.which_auth_header(f), None) - if not auth_value: - return False - parts = parse_http_basic_auth(auth_value) - if not parts: - return False - scheme, username, password = parts - if scheme.lower() != 'basic': - return False + def check(self, f: http.HTTPFlow) -> Optional[Tuple[str, str]]: + """ + Check if a request is correctly authenticated. + Returns: + - a (username, password) tuple if successful, + - None, otherwise. + """ + auth_value = f.request.headers.get(self.which_auth_header(), "") + try: + scheme, username, password = parse_http_basic_auth(auth_value) + except ValueError: + return None if self.nonanonymous: - pass + return username, password elif self.singleuser: - if [username, password] != self.singleuser: - return False + if self.singleuser == [username, password]: + return username, password elif self.htpasswd: - if not self.htpasswd.check_password(username, password): - return False - else: - raise NotImplementedError("Should never happen.") + if self.htpasswd.check_password(username, password): + return username, password - return True + return None - def authenticate(self, f): - if self.check(f): - del f.request.headers[self.which_auth_header(f)] + def authenticate(self, f: http.HTTPFlow) -> bool: + valid_credentials = self.check(f) + if valid_credentials: + f.metadata["proxyauth"] = valid_credentials + del f.request.headers[self.which_auth_header()] + return True else: - f.response = self.auth_required_response(f) + f.response = self.auth_required_response() + return False # Handlers def configure(self, options, updated): @@ -125,24 +138,28 @@ class ProxyAuth: ) else: self.htpasswd = None + if "mode" in updated: + self.mode = options.mode if self.enabled(): if options.mode == "transparent": raise exceptions.OptionsError( "Proxy Authentication not supported in transparent mode." ) - elif options.mode == "socks5": + if options.mode == "socks5": raise exceptions.OptionsError( "Proxy Authentication not supported in SOCKS mode. " "https://github.com/mitmproxy/mitmproxy/issues/738" ) - # TODO: check for multiple auth options + # TODO: check for multiple auth options - def http_connect(self, f): - if self.enabled() and f.mode == "regular": - self.authenticate(f) + def http_connect(self, f: http.HTTPFlow) -> None: + if self.enabled(): + if self.authenticate(f): + self.authenticated.add(f.client_conn) - def requestheaders(self, f): + def requestheaders(self, f: http.HTTPFlow) -> None: if self.enabled(): - # Are we already authenticated in CONNECT? - if not (f.mode == "regular" and f.server_conn.via): - self.authenticate(f) + # Is this connection authenticated by a previous HTTP CONNECT? + if f.client_conn in self.authenticated: + return + self.authenticate(f) diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 357172e3..c7db6690 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -159,6 +159,7 @@ def get_content_view(viewmode: View, data: bytes, **metadata): return desc, safe_to_print(content), error +# The order in which ContentViews are added is important! add(auto.ViewAuto()) add(raw.ViewRaw()) add(hex.ViewHex()) @@ -172,9 +173,7 @@ add(urlencoded.ViewURLEncoded()) add(multipart.ViewMultipart()) add(image.ViewImage()) add(query.ViewQuery()) - -if protobuf.ViewProtobuf.is_available(): - add(protobuf.ViewProtobuf()) +add(protobuf.ViewProtobuf()) __all__ = [ "View", "VIEW_CUTOFF", "KEY_MAX", "format_text", "format_dict", diff --git a/mitmproxy/contentviews/auto.py b/mitmproxy/contentviews/auto.py index 7b3cbd78..d46a1bd3 100644 --- a/mitmproxy/contentviews/auto.py +++ b/mitmproxy/contentviews/auto.py @@ -18,6 +18,8 @@ class ViewAuto(base.View): return contentviews.content_types_map[ct][0](data, **metadata) elif strutils.is_xml(data): return contentviews.get("XML/HTML")(data, **metadata) + elif ct.startswith("image/"): + return contentviews.get("Image")(data, **metadata) if metadata.get("query"): return contentviews.get("Query")(data, **metadata) if data and strutils.is_mostly_bin(data): diff --git a/mitmproxy/contentviews/image/__init__.py b/mitmproxy/contentviews/image/__init__.py index 0d0f06e0..33356bd7 100644 --- a/mitmproxy/contentviews/image/__init__.py +++ b/mitmproxy/contentviews/image/__init__.py @@ -1 +1,3 @@ -from .view import ViewImage # noqa +from .view import ViewImage + +__all__ = ["ViewImage"] diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index 1ff3cff7..062fb38e 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -13,9 +13,9 @@ Metadata = typing.List[typing.Tuple[str, str]] def parse_png(data: bytes) -> Metadata: img = png.Png(KaitaiStream(io.BytesIO(data))) parts = [ - ('Format', 'Portable network graphics') + ('Format', 'Portable network graphics'), + ('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height)) ] - parts.append(('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height))) for chunk in img.chunks: if chunk.type == 'gAMA': parts.append(('gamma', str(chunk.body.gamma_int / 100000))) @@ -34,13 +34,13 @@ def parse_png(data: bytes) -> Metadata: def parse_gif(data: bytes) -> Metadata: img = gif.Gif(KaitaiStream(io.BytesIO(data))) + descriptor = img.logical_screen_descriptor parts = [ - ('Format', 'Compuserve GIF') + ('Format', 'Compuserve GIF'), + ('Version', "GIF{}".format(img.header.version.decode('ASCII'))), + ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)), + ('background', str(descriptor.bg_color_index)) ] - parts.append(('version', "GIF{0}".format(img.header.version.decode('ASCII')))) - descriptor = img.logical_screen_descriptor - parts.append(('Size', "{0} x {1} px".format(descriptor.screen_width, descriptor.screen_height))) - parts.append(('background', str(descriptor.bg_color_index))) ext_blocks = [] for block in img.blocks: if block.block_type.name == 'extension': diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index 8fdb26e9..95ee1e43 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -1,55 +1,38 @@ -import io import imghdr -from PIL import Image - +from mitmproxy.contentviews import base from mitmproxy.types import multidict from . import image_parser -from mitmproxy.contentviews import base - class ViewImage(base.View): name = "Image" prompt = ("image", "i") + + # there is also a fallback in the auto view for image/*. content_types = [ "image/png", "image/jpeg", "image/gif", "image/vnd.microsoft.icon", "image/x-icon", + "image/webp", ] def __call__(self, data, **metadata): image_type = imghdr.what('', h=data) if image_type == 'png': - f = "PNG" - parts = image_parser.parse_png(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_png(data) elif image_type == 'gif': - f = "GIF" - parts = image_parser.parse_gif(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_gif(data) elif image_type == 'jpeg': - f = "JPEG" - parts = image_parser.parse_jpeg(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt - try: - img = Image.open(io.BytesIO(data)) - except IOError: - return None - parts = [ - ("Format", str(img.format_description)), - ("Size", "%s x %s px" % img.size), - ("Mode", str(img.mode)), - ] - for i in sorted(img.info.keys()): - if i != "exif": - parts.append( - (str(i), str(img.info[i])) - ) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % img.format, fmt + image_metadata = image_parser.parse_jpeg(data) + else: + image_metadata = [ + ("Image Format", image_type or "unknown") + ] + if image_type: + view_name = "{} Image".format(image_type.upper()) + else: + view_name = "Unknown Image" + return view_name, base.format_dict(multidict.MultiDict(image_metadata)) diff --git a/mitmproxy/contentviews/protobuf.py b/mitmproxy/contentviews/protobuf.py index 620d9444..4bbb1580 100644 --- a/mitmproxy/contentviews/protobuf.py +++ b/mitmproxy/contentviews/protobuf.py @@ -15,31 +15,28 @@ class ViewProtobuf(base.View): "application/x-protobuffer", ] - @staticmethod - def is_available(): + def is_available(self): try: p = subprocess.Popen( ["protoc", "--version"], stdout=subprocess.PIPE ) out, _ = p.communicate() - return out.startswith("libprotoc") + return out.startswith(b"libprotoc") except: return False - def decode_protobuf(self, content): + def __call__(self, data, **metadata): + if not self.is_available(): + raise NotImplementedError("protoc not found. Please make sure 'protoc' is available in $PATH.") + # if Popen raises OSError, it will be caught in # get_content_view and fall back to Raw p = subprocess.Popen(['protoc', '--decode_raw'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate(input=content) - if out: - return out - else: - return err - - def __call__(self, data, **metadata): - decoded = self.decode_protobuf(data) + decoded, _ = p.communicate(input=data) + if not decoded: + raise ValueError("Failed to parse input.") return "Protobuf", base.format_text(decoded) diff --git a/mitmproxy/export.py b/mitmproxy/export.py index 0a261509..235e754a 100644 --- a/mitmproxy/export.py +++ b/mitmproxy/export.py @@ -73,7 +73,7 @@ def python_code(flow: http.HTTPFlow): headers = flow.request.headers.copy() # requests adds those by default. - for x in ("host", "content-length"): + for x in (":authority", "host", "content-length"): headers.pop(x, None) writearg("headers", dict(headers)) try: @@ -130,7 +130,7 @@ def locust_code(flow): if flow.request.headers: lines = [ (_native(k), _native(v)) for k, v in flow.request.headers.fields - if _native(k).lower() not in ["host", "cookie"] + if _native(k).lower() not in [":authority", "host", "cookie"] ] lines = [" '%s': '%s',\n" % (k, v) for k, v in lines] headers += "\n headers = {\n%s }\n" % "".join(lines) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 9c59984a..f0cabcf8 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,4 +1,5 @@ -import cgi +import html +from typing import Optional from mitmproxy import flow @@ -203,16 +204,27 @@ class HTTPFlow(flow.Flow): return c -def make_error_response(status_code, message, headers=None): - response = http.status_codes.RESPONSES.get(status_code, "Unknown") +def make_error_response( + status_code: int, + message: str="", + headers: Optional[http.Headers]=None, +) -> HTTPResponse: + reason = http.status_codes.RESPONSES.get(status_code, "Unknown") body = """ <html> <head> - <title>%d %s</title> + <title>{status_code} {reason}</title> </head> - <body>%s</body> + <body> + <h1>{status_code} {reason}</h1> + <p>{message}</p> + </body> </html> - """.strip() % (status_code, response, cgi.escape(message)) + """.strip().format( + status_code=status_code, + reason=reason, + message=html.escape(message), + ) body = body.encode("utf8", "replace") if not headers: @@ -226,7 +238,7 @@ def make_error_response(status_code, message, headers=None): return HTTPResponse( b"HTTP/1.1", status_code, - response, + reason, headers, body, ) diff --git a/mitmproxy/net/http/http1/assemble.py b/mitmproxy/net/http/http1/assemble.py index d718589f..8b7246f7 100644 --- a/mitmproxy/net/http/http1/assemble.py +++ b/mitmproxy/net/http/http1/assemble.py @@ -78,8 +78,9 @@ def _assemble_request_headers(request_data): Args: request_data (mitmproxy.net.http.request.RequestData) """ - headers = request_data.headers.copy() + headers = request_data.headers if "host" not in headers and request_data.scheme and request_data.host and request_data.port: + headers = headers.copy() headers["host"] = mitmproxy.net.http.url.hostport( request_data.scheme, request_data.host, diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index d0493da4..ef88fd6c 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -227,7 +227,7 @@ def _get_first_line(rfile): if line == b"\r\n" or line == b"\n": # Possible leftover from previous message line = rfile.readline() - except exceptions.TcpDisconnect: + except (exceptions.TcpDisconnect, exceptions.TlsException): raise exceptions.HttpReadDisconnect("Remote disconnected") if not line: raise exceptions.HttpReadDisconnect("Remote disconnected") diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 822f8229..b961e1e4 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +from typing import Optional from mitmproxy.types import multidict from mitmproxy.utils import strutils @@ -164,11 +165,44 @@ class Request(message.Message): self.data.host = host # Update host header - if "host" in self.headers: - if host: - self.headers["host"] = host + if self.host_header is not None: + self.host_header = host + + @property + def host_header(self) -> Optional[str]: + """ + The request's host/authority header. + + This property maps to either ``request.headers["Host"]`` or + ``request.headers[":authority"]``, depending on whether it's HTTP/1.x or HTTP/2.0. + """ + if ":authority" in self.headers: + return self.headers[":authority"] + if "Host" in self.headers: + return self.headers["Host"] + return None + + @host_header.setter + def host_header(self, val: Optional[str]) -> None: + if val is None: + self.headers.pop("Host", None) + self.headers.pop(":authority", None) + elif self.host_header is not None: + # Update any existing headers. + if ":authority" in self.headers: + self.headers[":authority"] = val + if "Host" in self.headers: + self.headers["Host"] = val + else: + # Only add the correct new header. + if self.http_version.upper().startswith("HTTP/2"): + self.headers[":authority"] = val else: - self.headers.pop("host") + self.headers["Host"] = val + + @host_header.deleter + def host_header(self): + self.host_header = None @property def port(self): @@ -211,9 +245,10 @@ class Request(message.Message): def _parse_host_header(self): """Extract the host and port from Host header""" - if "host" not in self.headers: + host = self.host_header + if not host: return None, None - host, port = self.headers["host"], None + port = None m = host_header_re.match(host) if m: host = m.group("host").strip("[]") @@ -373,7 +408,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = mitmproxy.net.http.url.encode(form_data).encode() + self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() @urlencoded_form.setter def urlencoded_form(self, value): diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index ff3d5264..86ce9764 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -82,11 +82,24 @@ def unparse(scheme, host, port, path=""): return "%s://%s%s" % (scheme, hostport(scheme, host, port), path) -def encode(s: Sequence[Tuple[str, str]]) -> str: +def encode(s: Sequence[Tuple[str, str]], similar_to: str=None) -> str: """ Takes a list of (key, value) tuples and returns a urlencoded string. + If similar_to is passed, the output is formatted similar to the provided urlencoded string. """ - return urllib.parse.urlencode(s, False, errors="surrogateescape") + + remove_trailing_equal = False + if similar_to: + remove_trailing_equal = any("=" not in param for param in similar_to.split("&")) + + encoded = urllib.parse.urlencode(s, False, errors="surrogateescape") + + if remove_trailing_equal: + encoded = encoded.replace("=&", "&") + if encoded[-1] == '=': + encoded = encoded[:-1] + + return encoded def decode(s): diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 630d2964..2467b9dd 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -54,6 +54,7 @@ class Options(optmanager.OptManager): server_replay_ignore_params: Sequence[str] = [], server_replay_ignore_payload_params: Sequence[str] = [], server_replay_ignore_host: bool = False, + # Proxy options auth_nonanonymous: bool = False, auth_singleuser: Optional[str] = None, @@ -65,15 +66,18 @@ class Options(optmanager.OptManager): ciphers_client: str=DEFAULT_CLIENT_CIPHERS, ciphers_server: Optional[str]=None, clientcerts: Optional[str] = None, - http2: bool = True, ignore_hosts: Sequence[str] = [], listen_host: str = "", listen_port: int = LISTEN_PORT, upstream_bind_address: str = "", mode: str = "regular", no_upstream_cert: bool = False, - rawtcp: bool = False, + + http2: bool = True, + http2_priority: bool = False, websocket: bool = True, + rawtcp: bool = False, + spoof_source_address: bool = False, upstream_server: Optional[str] = None, upstream_auth: Optional[str] = None, @@ -152,15 +156,18 @@ class Options(optmanager.OptManager): self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts - self.http2 = http2 self.ignore_hosts = ignore_hosts self.listen_host = listen_host self.listen_port = listen_port self.upstream_bind_address = upstream_bind_address self.mode = mode self.no_upstream_cert = no_upstream_cert - self.rawtcp = rawtcp + + self.http2 = http2 + self.http2_priority = http2_priority self.websocket = websocket + self.rawtcp = rawtcp + self.spoof_source_address = spoof_source_address self.upstream_server = upstream_server self.upstream_auth = upstream_auth diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index da9a8781..a7d56f24 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -291,7 +291,7 @@ class HttpLayer(base.Layer): # update host header in reverse proxy mode if self.config.options.mode == "reverse": - f.request.headers["Host"] = self.config.upstream_server.address.host + f.request.host_header = self.config.upstream_server.address.host # Determine .scheme, .host and .port attributes for inline scripts. For # absolute-form requests, they are directly given in the request. For @@ -301,11 +301,10 @@ class HttpLayer(base.Layer): if self.mode is HTTPMode.transparent: # Setting request.host also updates the host header, which we want # to preserve - host_header = f.request.headers.get("host", None) + host_header = f.request.host_header f.request.host = self.__initial_server_conn.address.host f.request.port = self.__initial_server_conn.address.port - if host_header: - f.request.headers["host"] = host_header + f.request.host_header = host_header # set again as .host overwrites this. f.request.scheme = "https" if self.__initial_server_tls else "http" self.channel.ask("request", f) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index b7548221..cdce24b3 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -268,6 +268,10 @@ class Http2Layer(base.Layer): return True def _handle_priority_updated(self, eid, event): + if not self.config.options.http2_priority: + self.log("HTTP/2 PRIORITY frame surpressed. Use --http2-priority to enable forwarding.", "debug") + return True + if eid in self.streams and self.streams[eid].handled_priority_event is event: # this event was already handled during stream creation # HeadersFrame + Priority information as RequestReceived @@ -527,9 +531,12 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr if self.handled_priority_event: # only send priority information if they actually came with the original HeadersFrame # and not if they got updated before/after with a PriorityFrame - priority_exclusive = self.priority_exclusive - priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) - priority_weight = self.priority_weight + if not self.config.options.http2_priority: + self.log("HTTP/2 PRIORITY information in HEADERS frame surpressed. Use --http2-priority to enable forwarding.", "debug") + else: + priority_exclusive = self.priority_exclusive + priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) + priority_weight = self.priority_weight try: self.connections[self.server_conn].safe_send_headers( @@ -610,7 +617,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr chunks ) - def __call__(self): + def __call__(self): # pragma: no cover raise EnvironmentError('Http2SingleStreamLayer must be run as thread') def run(self): diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 1c620fd6..bb0bb17a 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -137,7 +137,6 @@ def get_common_options(args): ciphers_client = args.ciphers_client, ciphers_server = args.ciphers_server, clientcerts = args.clientcerts, - http2 = args.http2, ignore_hosts = args.ignore_hosts, listen_host = args.addr, listen_port = args.port, @@ -145,8 +144,12 @@ def get_common_options(args): mode = mode, no_upstream_cert = args.no_upstream_cert, spoof_source_address = args.spoof_source_address, - rawtcp = args.rawtcp, + + http2 = args.http2, + http2_priority = args.http2_priority, websocket = args.websocket, + rawtcp = args.rawtcp, + upstream_server = upstream_server, upstream_auth = args.upstream_auth, ssl_version_client = args.ssl_version_client, @@ -334,18 +337,26 @@ def proxy_options(parser): ) http2 = group.add_mutually_exclusive_group() - http2.add_argument("--no-http2", action="store_false", dest="http2", + http2.add_argument("--no-http2", action="store_false", dest="http2") + http2.add_argument("--http2", action="store_true", dest="http2", help="Explicitly enable/disable HTTP/2 support. " - "Enabled by default." + "HTTP/2 support is enabled by default.", ) - http2.add_argument("--http2", action="store_true", dest="http2") + + http2_priority = group.add_mutually_exclusive_group() + http2_priority.add_argument("--http2-priority", action="store_true", dest="http2_priority") + http2_priority.add_argument("--no-http2-priority", action="store_false", dest="http2_priority", + help="Explicitly enable/disable PRIORITY forwarding for HTTP/2 connections. " + "PRIORITY forwarding is disabled by default, " + "because some webservers fail at implementing the RFC properly.", + ) websocket = group.add_mutually_exclusive_group() - websocket.add_argument("--no-websocket", action="store_false", dest="websocket", + websocket.add_argument("--no-websocket", action="store_false", dest="websocket") + websocket.add_argument("--websocket", action="store_true", dest="websocket", help="Explicitly enable/disable WebSocket support. " - "Enabled by default." + "WebSocket support is enabled by default.", ) - websocket.add_argument("--websocket", action="store_true", dest="websocket") parser.add_argument( "--upstream-auth", diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index a33a9823..4ab9e1f4 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -412,7 +412,7 @@ class ConsoleMaster(master.Master): def websocket_message(self, f): super().websocket_message(f) message = f.messages[-1] - signals.add_log(message.info, "info") + signals.add_log(f.message_info(message), "info") signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") @controller.handler diff --git a/mitmproxy/utils/debug.py b/mitmproxy/utils/debug.py index ff98b86c..c2eee2b6 100644 --- a/mitmproxy/utils/debug.py +++ b/mitmproxy/utils/debug.py @@ -37,8 +37,12 @@ def dump_system_info(): except: pass + bin_indicator = "" # PyInstaller builds indicator, if using precompiled binary + if getattr(sys, 'frozen', False): + bin_indicator = "Precompiled Binary" + data = [ - "Mitmproxy version: {} ({})".format(version.VERSION, git_describe), + "Mitmproxy version: {} ({}) {}".format(version.VERSION, git_describe, bin_indicator), "Python version: {}".format(platform.python_version()), "Platform: {}".format(platform.platform()), "SSL version: {}".format(SSL.SSLeay_version(SSL.SSLEAY_VERSION).decode()), diff --git a/pathod/pathoc.py b/pathod/pathoc.py index aba5c344..549444ca 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -11,15 +11,14 @@ import time import OpenSSL.crypto import logging -from mitmproxy.test.tutils import treq -from mitmproxy.utils import strutils -from mitmproxy.net import tcp from mitmproxy import certs +from mitmproxy import exceptions +from mitmproxy.net import tcp from mitmproxy.net import websockets from mitmproxy.net import socks -from mitmproxy import exceptions -from mitmproxy.net.http import http1 +from mitmproxy.net import http as net_http from mitmproxy.types import basethread +from mitmproxy.utils import strutils from pathod import log from pathod import language @@ -234,7 +233,7 @@ class Pathoc(tcp.TCPClient): ) self.protocol = http2.HTTP2StateProtocol(self, dump_frames=self.http2_framedump) else: - self.protocol = http1 + self.protocol = net_http.http1 self.settings = language.Settings( is_client=True, @@ -245,13 +244,20 @@ class Pathoc(tcp.TCPClient): ) def http_connect(self, connect_to): - self.wfile.write( - b'CONNECT %s:%d HTTP/1.1\r\n' % (connect_to[0].encode("idna"), connect_to[1]) + - b'\r\n' + req = net_http.Request( + first_line_format='authority', + method='CONNECT', + scheme=None, + host=connect_to[0].encode("idna"), + port=connect_to[1], + path=None, + http_version='HTTP/1.1', + content=b'', ) + self.wfile.write(net_http.http1.assemble_request(req)) self.wfile.flush() try: - resp = self.protocol.read_response(self.rfile, treq(method=b"CONNECT")) + resp = self.protocol.read_response(self.rfile, req) if resp.status_code != 200: raise exceptions.HttpException("Unexpected status code: %s" % resp.status_code) except exceptions.HttpException as e: @@ -435,7 +441,20 @@ class Pathoc(tcp.TCPClient): req = language.serve(r, self.wfile, self.settings) self.wfile.flush() - resp = self.protocol.read_response(self.rfile, treq(method=req["method"].encode())) + # build a dummy request to read the reponse + # ideally this would be returned directly from language.serve + dummy_req = net_http.Request( + first_line_format="relative", + method=req["method"], + scheme=b"http", + host=b"localhost", + port=80, + path=b"/", + http_version=b"HTTP/1.1", + content=b'', + ) + + resp = self.protocol.read_response(self.rfile, dummy_req) resp.sslinfo = self.sslinfo except exceptions.HttpException as v: lg("Invalid server response: %s" % v) @@ -18,3 +18,95 @@ show_missing = True exclude_lines = pragma: no cover raise NotImplementedError() + +[tool:full_coverage] +exclude = + mitmproxy/contentviews/__init__.py + mitmproxy/contentviews/wbxml.py + mitmproxy/contentviews/xml_html.py + mitmproxy/net/tcp.py + mitmproxy/net/http/cookies.py + mitmproxy/net/http/encoding.py + mitmproxy/net/http/message.py + mitmproxy/net/http/url.py + mitmproxy/proxy/protocol/ + mitmproxy/proxy/config.py + mitmproxy/proxy/root_context.py + mitmproxy/proxy/server.py + mitmproxy/tools/ + mitmproxy/certs.py + mitmproxy/connections.py + mitmproxy/controller.py + mitmproxy/export.py + mitmproxy/flow.py + mitmproxy/flowfilter.py + mitmproxy/http.py + mitmproxy/io_compat.py + mitmproxy/master.py + mitmproxy/optmanager.py + pathod/pathoc.py + pathod/pathod.py + pathod/test.py + pathod/protocols/http2.py + +[tool:individual_coverage] +exclude = + mitmproxy/addonmanager.py + mitmproxy/addons/onboardingapp/app.py + mitmproxy/addons/termlog.py + mitmproxy/certs.py + mitmproxy/connections.py + mitmproxy/contentviews/base.py + mitmproxy/contentviews/wbxml.py + mitmproxy/contentviews/xml_html.py + mitmproxy/controller.py + mitmproxy/ctx.py + mitmproxy/exceptions.py + mitmproxy/export.py + mitmproxy/flow.py + mitmproxy/flowfilter.py + mitmproxy/http.py + mitmproxy/io.py + mitmproxy/io_compat.py + mitmproxy/log.py + mitmproxy/master.py + mitmproxy/net/check.py + mitmproxy/net/http/cookies.py + mitmproxy/net/http/headers.py + mitmproxy/net/http/message.py + mitmproxy/net/http/multipart.py + mitmproxy/net/http/url.py + mitmproxy/net/tcp.py + mitmproxy/options.py + mitmproxy/optmanager.py + mitmproxy/proxy/config.py + mitmproxy/proxy/modes/http_proxy.py + mitmproxy/proxy/modes/reverse_proxy.py + mitmproxy/proxy/modes/socks_proxy.py + mitmproxy/proxy/modes/transparent_proxy.py + mitmproxy/proxy/protocol/base.py + mitmproxy/proxy/protocol/http.py + mitmproxy/proxy/protocol/http1.py + mitmproxy/proxy/protocol/http2.py + mitmproxy/proxy/protocol/http_replay.py + mitmproxy/proxy/protocol/rawtcp.py + mitmproxy/proxy/protocol/tls.py + mitmproxy/proxy/protocol/websocket.py + mitmproxy/proxy/root_context.py + mitmproxy/proxy/server.py + mitmproxy/stateobject.py + mitmproxy/types/multidict.py + mitmproxy/utils/bits.py + pathod/language/actions.py + pathod/language/base.py + pathod/language/exceptions.py + pathod/language/generators.py + pathod/language/http.py + pathod/language/message.py + pathod/log.py + pathod/pathoc.py + pathod/pathod.py + pathod/protocols/http.py + pathod/protocols/http2.py + pathod/protocols/websockets.py + pathod/test.py @@ -37,7 +37,6 @@ setup( "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Security", "Topic :: Internet", "Topic :: Internet :: WWW/HTTP", @@ -72,7 +71,6 @@ setup( "hyperframe>=4.0.1, <5", "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.6, <0.7", - "Pillow>=3.2, <4.1", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", "pyOpenSSL>=16.0, <17.0", @@ -112,13 +110,11 @@ setup( "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ - "protobuf>=3.1.0, <3.3", - # TODO: Find Python 3 replacement - # "pyamf>=0.8.0, <0.9", ], 'examples': [ "beautifulsoup4>=4.4.1, <4.6", "pytz>=2015.07.0, <=2016.10", + "Pillow>=3.2, <4.1", ] } ) diff --git a/test/conftest.py b/test/conftest.py index 83823a19..b4e1da93 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,6 +6,7 @@ from contextlib import contextmanager import mitmproxy.net.tcp +pytest_plugins = ('test.full_coverage_plugin',) requires_alpn = pytest.mark.skipif( not mitmproxy.net.tcp.HAS_ALPN, @@ -27,10 +28,17 @@ skip_appveyor = pytest.mark.skipif( ) -original_pytest_raises = pytest.raises +@pytest.fixture() +def disable_alpn(monkeypatch): + monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False) + monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False) +################################################################################ # TODO: remove this wrapper when pytest 3.1.0 is released +original_pytest_raises = pytest.raises + + @contextmanager @functools.wraps(original_pytest_raises) def raises(exc, *args, **kwargs): @@ -41,116 +49,4 @@ def raises(exc, *args, **kwargs): pytest.raises = raises - - -@pytest.fixture() -def disable_alpn(monkeypatch): - monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False) - monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False) - - -enable_coverage = False -coverage_values = [] -coverage_passed = False - - -def pytest_addoption(parser): - parser.addoption('--full-cov', - action='append', - dest='full_cov', - default=[], - help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none") - - parser.addoption('--no-full-cov', - action='append', - dest='no_full_cov', - default=[], - help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none") - - -def pytest_configure(config): - global enable_coverage - enable_coverage = ( - len(config.getoption('file_or_dir')) == 0 and - len(config.getoption('full_cov')) > 0 and - config.pluginmanager.getplugin("_cov") is not None and - config.pluginmanager.getplugin("_cov").cov_controller is not None and - config.pluginmanager.getplugin("_cov").cov_controller.cov is not None - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtestloop(session): - global enable_coverage - global coverage_values - global coverage_passed - - if not enable_coverage: - yield - return - - cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov - - if os.name == 'nt': - cov.exclude('pragma: windows no cover') - - yield - - coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) - - prefix = os.getcwd() - excluded_files = [os.path.normpath(f) for f in pytest.config.option.no_full_cov] - measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()] - measured_files = [f for f in measured_files if not any(f.startswith(excluded_f) for excluded_f in excluded_files)] - - for name in coverage_values.keys(): - files = [f for f in measured_files if f.startswith(os.path.normpath(name))] - try: - with open(os.devnull, 'w') as null: - overall = cov.report(files, ignore_errors=True, file=null) - singles = [(s, cov.report(s, ignore_errors=True, file=null)) for s in files] - coverage_values[name] = (overall, singles) - except: - pass - - if any(v < 100 for v, _ in coverage_values.values()): - # make sure we get the EXIT_TESTSFAILED exit code - session.testsfailed += 1 - else: - coverage_passed = True - - -def pytest_terminal_summary(terminalreporter, exitstatus): - global enable_coverage - global coverage_values - global coverage_passed - - if not enable_coverage: - return - - terminalreporter.write('\n') - if not coverage_passed: - markup = {'red': True, 'bold': True} - msg = "FAIL: Full test coverage not reached!\n" - terminalreporter.write(msg, **markup) - - for name in sorted(coverage_values.keys()): - msg = 'Coverage for {}: {:.2f}%\n'.format(name, coverage_values[name][0]) - if coverage_values[name][0] < 100: - markup = {'red': True, 'bold': True} - for s, v in sorted(coverage_values[name][1]): - if v < 100: - msg += ' {}: {:.2f}%\n'.format(s, v) - else: - markup = {'green': True} - terminalreporter.write(msg, **markup) - else: - markup = {'green': True} - msg = 'SUCCESS: Full test coverage reached in modules and files:\n' - msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) - terminalreporter.write(msg, **markup) - - msg = '\nExcluded files:\n' - for s in sorted(pytest.config.option.no_full_cov): - msg += " {}\n".format(s) - terminalreporter.write(msg) +################################################################################ diff --git a/test/filename_matching.py b/test/filename_matching.py new file mode 100644 index 00000000..51cedf03 --- /dev/null +++ b/test/filename_matching.py @@ -0,0 +1,57 @@ +import os +import re +import glob +import sys + + +def check_src_files_have_test(): + missing_test_files = [] + + excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] + src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) + src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] + src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] + for f in src_files: + p = os.path.join("test", os.path.dirname(f), "test_" + os.path.basename(f)) + if not os.path.isfile(p): + missing_test_files.append((f, p)) + + return missing_test_files + + +def check_test_files_have_src(): + unknown_test_files = [] + + excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py'] + test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) + glob.glob('test/pathod/**/*.py', recursive=True) + test_files = [f for f in test_files if os.path.basename(f) != '__init__.py'] + test_files = [f for f in test_files if not any(os.path.normpath(p) in f for p in excluded)] + for f in test_files: + p = os.path.join(re.sub('^test/', '', os.path.dirname(f)), re.sub('^test_', '', os.path.basename(f))) + if not os.path.isfile(p): + unknown_test_files.append((f, p)) + + return unknown_test_files + + +def main(): + exitcode = 0 + + missing_test_files = check_src_files_have_test() + if missing_test_files: + exitcode += 1 + for f, p in sorted(missing_test_files): + print("{} MUST have a matching test file: {}".format(f, p)) + + unknown_test_files = check_test_files_have_src() + if unknown_test_files: + # TODO: enable this in the future + # exitcode += 1 + for f, p in sorted(unknown_test_files): + print("{} DOES NOT MATCH a source file! Expected to find: {}".format(f, p)) + + sys.exit(exitcode) + + +if __name__ == '__main__': + main() diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py new file mode 100644 index 00000000..d98c29d6 --- /dev/null +++ b/test/full_coverage_plugin.py @@ -0,0 +1,121 @@ +import os +import configparser +import pytest + +here = os.path.abspath(os.path.dirname(__file__)) + + +enable_coverage = False +coverage_values = [] +coverage_passed = True +no_full_cov = [] + + +def pytest_addoption(parser): + parser.addoption('--full-cov', + action='append', + dest='full_cov', + default=[], + help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none") + + parser.addoption('--no-full-cov', + action='append', + dest='no_full_cov', + default=[], + help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none") + + +def pytest_configure(config): + global enable_coverage + global no_full_cov + + enable_coverage = ( + len(config.getoption('file_or_dir')) == 0 and + len(config.getoption('full_cov')) > 0 and + config.pluginmanager.getplugin("_cov") is not None and + config.pluginmanager.getplugin("_cov").cov_controller is not None and + config.pluginmanager.getplugin("_cov").cov_controller.cov is not None + ) + + c = configparser.ConfigParser() + c.read(os.path.join(here, "..", "setup.cfg")) + fs = c['tool:full_coverage']['exclude'].split('\n') + no_full_cov = config.option.no_full_cov + [f.strip() for f in fs] + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtestloop(session): + global enable_coverage + global coverage_values + global coverage_passed + global no_full_cov + + if not enable_coverage: + yield + return + + cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov + + if os.name == 'nt': + cov.exclude('pragma: windows no cover') + + yield + + coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) + + prefix = os.getcwd() + + excluded_files = [os.path.normpath(f) for f in no_full_cov] + measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()] + measured_files = [f for f in measured_files if not any(f.startswith(excluded_f) for excluded_f in excluded_files)] + + for name in coverage_values.keys(): + files = [f for f in measured_files if f.startswith(os.path.normpath(name))] + try: + with open(os.devnull, 'w') as null: + overall = cov.report(files, ignore_errors=True, file=null) + singles = [(s, cov.report(s, ignore_errors=True, file=null)) for s in files] + coverage_values[name] = (overall, singles) + except: + pass + + if any(v < 100 for v, _ in coverage_values.values()): + # make sure we get the EXIT_TESTSFAILED exit code + session.testsfailed += 1 + coverage_passed = False + + +def pytest_terminal_summary(terminalreporter, exitstatus): + global enable_coverage + global coverage_values + global coverage_passed + global no_full_cov + + if not enable_coverage: + return + + terminalreporter.write('\n') + if not coverage_passed: + markup = {'red': True, 'bold': True} + msg = "FAIL: Full test coverage not reached!\n" + terminalreporter.write(msg, **markup) + + for name in sorted(coverage_values.keys()): + msg = 'Coverage for {}: {:.2f}%\n'.format(name, coverage_values[name][0]) + if coverage_values[name][0] < 100: + markup = {'red': True, 'bold': True} + for s, v in sorted(coverage_values[name][1]): + if v < 100: + msg += ' {}: {:.2f}%\n'.format(s, v) + else: + markup = {'green': True} + terminalreporter.write(msg, **markup) + else: + msg = 'SUCCESS: Full test coverage reached in modules and files:\n' + msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) + terminalreporter.write(msg, green=True) + + msg = '\nExcluded files:\n' + for s in sorted(no_full_cov): + msg += " {}\n".format(s) + terminalreporter.write(msg) diff --git a/test/mitmproxy/tools/1024example b/test/helper_tools/1024example index 78af7ed0..78af7ed0 100644 --- a/test/mitmproxy/tools/1024example +++ b/test/helper_tools/1024example diff --git a/test/mitmproxy/tools/ab.exe b/test/helper_tools/ab.exe Binary files differindex d68ed0f3..d68ed0f3 100644 --- a/test/mitmproxy/tools/ab.exe +++ b/test/helper_tools/ab.exe diff --git a/test/mitmproxy/tools/bench.py b/test/helper_tools/bench.py index fb75ef46..fb75ef46 100644 --- a/test/mitmproxy/tools/bench.py +++ b/test/helper_tools/bench.py diff --git a/test/mitmproxy/tools/benchtool.py b/test/helper_tools/benchtool.py index b9078d0e..b9078d0e 100644 --- a/test/mitmproxy/tools/benchtool.py +++ b/test/helper_tools/benchtool.py diff --git a/test/mitmproxy/addons/dumperview.py b/test/helper_tools/dumperview.py index be56fe14..be56fe14 100755 --- a/test/mitmproxy/addons/dumperview.py +++ b/test/helper_tools/dumperview.py diff --git a/test/mitmproxy/tools/getcert b/test/helper_tools/getcert index 43ebf11d..43ebf11d 100644 --- a/test/mitmproxy/tools/getcert +++ b/test/helper_tools/getcert diff --git a/test/mitmproxy/tools/inspect_dumpfile.py b/test/helper_tools/inspect_dumpfile.py index b2201f40..b2201f40 100644 --- a/test/mitmproxy/tools/inspect_dumpfile.py +++ b/test/helper_tools/inspect_dumpfile.py diff --git a/test/mitmproxy/tools/memoryleak.py b/test/helper_tools/memoryleak.py index c03230da..c03230da 100644 --- a/test/mitmproxy/tools/memoryleak.py +++ b/test/helper_tools/memoryleak.py diff --git a/test/mitmproxy/tools/passive_close.py b/test/helper_tools/passive_close.py index 6f97ea4f..6f97ea4f 100644 --- a/test/mitmproxy/tools/passive_close.py +++ b/test/helper_tools/passive_close.py diff --git a/test/mitmproxy/tools/testpatt b/test/helper_tools/testpatt index b41011c0..b41011c0 100644 --- a/test/mitmproxy/tools/testpatt +++ b/test/helper_tools/testpatt diff --git a/test/individual_coverage.py b/test/individual_coverage.py new file mode 100644 index 00000000..35bcd27f --- /dev/null +++ b/test/individual_coverage.py @@ -0,0 +1,82 @@ +import io +import contextlib +import os +import sys +import glob +import multiprocessing +import configparser +import itertools +import pytest + + +def run_tests(src, test, fail): + stderr = io.StringIO() + stdout = io.StringIO() + with contextlib.redirect_stderr(stderr): + with contextlib.redirect_stdout(stdout): + e = pytest.main([ + '-qq', + '--disable-pytest-warnings', + '--no-faulthandler', + '--cov', src.replace('.py', '').replace('/', '.'), + '--cov-fail-under', '100', + '--cov-report', 'term-missing:skip-covered', + test + ]) + + if e == 0: + if fail: + print("SUCCESS but should have FAILED:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") + e = 42 + else: + print("SUCCESS:", src) + else: + if fail: + print("Ignoring fail:", src) + e = 0 + else: + cov = [l for l in stdout.getvalue().split("\n") if (src in l) or ("was never imported" in l)] + if len(cov) == 1: + print("FAIL:", cov[0]) + else: + print("FAIL:", src, test, stdout.getvalue(), stdout.getvalue()) + print(stderr.getvalue()) + print(stdout.getvalue()) + + sys.exit(e) + + +def start_pytest(src, test, fail): + # run pytest in a new process, otherwise imports and modules might conflict + proc = multiprocessing.Process(target=run_tests, args=(src, test, fail)) + proc.start() + proc.join() + return (src, test, proc.exitcode) + + +def main(): + c = configparser.ConfigParser() + c.read('setup.cfg') + fs = c['tool:individual_coverage']['exclude'].strip().split('\n') + no_individual_cov = [f.strip() for f in fs] + + excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] + src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) + src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] + src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] + + ps = [] + for src in sorted(src_files): + test = os.path.join("test", os.path.dirname(src), "test_" + os.path.basename(src)) + if os.path.isfile(test): + ps.append((src, test, src in no_individual_cov)) + + result = list(itertools.starmap(start_pytest, ps)) + + if any(e != 0 for _, _, e in result): + sys.exit(1) + pass + + +if __name__ == '__main__': + main() diff --git a/test/mitmproxy/__init__.py b/test/mitmproxy/__init__.py index 28dc133f..6f114e18 100644 --- a/test/mitmproxy/__init__.py +++ b/test/mitmproxy/__init__.py @@ -3,5 +3,4 @@ import logging logging.getLogger("hyper").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("passlib").setLevel(logging.WARNING) -logging.getLogger("PIL").setLevel(logging.WARNING) logging.getLogger("tornado").setLevel(logging.WARNING) diff --git a/test/mitmproxy/addons/onboardingapp/test_app.py b/test/mitmproxy/addons/onboardingapp/test_app.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/addons/onboardingapp/test_app.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/addons/test_evenstore.py b/test/mitmproxy/addons/test_eventstore.py index f54b9980..f54b9980 100644 --- a/test/mitmproxy/addons/test_evenstore.py +++ b/test/mitmproxy/addons/test_eventstore.py diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index b59b87c1..dd5829ab 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -1,21 +1,27 @@ import binascii + import pytest from mitmproxy import exceptions +from mitmproxy.addons import proxyauth from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.addons import proxyauth def test_parse_http_basic_auth(): assert proxyauth.parse_http_basic_auth( proxyauth.mkauth("test", "test") ) == ("basic", "test", "test") - assert not proxyauth.parse_http_basic_auth("") - assert not proxyauth.parse_http_basic_auth("foo bar") - v = "basic " + binascii.b2a_base64(b"foo").decode("ascii") - assert not proxyauth.parse_http_basic_auth(v) + with pytest.raises(ValueError): + proxyauth.parse_http_basic_auth("") + with pytest.raises(ValueError): + proxyauth.parse_http_basic_auth("foo bar") + with pytest.raises(ValueError): + proxyauth.parse_http_basic_auth("basic abc") + with pytest.raises(ValueError): + v = "basic " + binascii.b2a_base64(b"foo").decode("ascii") + proxyauth.parse_http_basic_auth(v) def test_configure(): @@ -42,14 +48,14 @@ def test_configure(): ctx.configure( up, - auth_htpasswd = tutils.test_data.path( + auth_htpasswd=tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) assert up.htpasswd assert up.htpasswd.check_password("test", "test") assert not up.htpasswd.check_password("test", "foo") - ctx.configure(up, auth_htpasswd = None) + ctx.configure(up, auth_htpasswd=None) assert not up.htpasswd with pytest.raises(exceptions.OptionsError): @@ -57,11 +63,14 @@ def test_configure(): with pytest.raises(exceptions.OptionsError): ctx.configure(up, auth_nonanonymous=True, mode="socks5") + ctx.configure(up, mode="regular") + assert up.mode == "regular" + def test_check(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True) + ctx.configure(up, auth_nonanonymous=True, mode="regular") f = tflow.tflow() assert not up.check(f) f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( @@ -73,7 +82,7 @@ def test_check(): assert not up.check(f) f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( - "test", "test", scheme = "unknown" + "test", "test", scheme="unknown" ) assert not up.check(f) @@ -87,8 +96,8 @@ def test_check(): ctx.configure( up, - auth_singleuser = None, - auth_htpasswd = tutils.test_data.path( + auth_singleuser=None, + auth_htpasswd=tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) @@ -105,7 +114,7 @@ def test_check(): def test_authenticate(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True) + ctx.configure(up, auth_nonanonymous=True, mode="regular") f = tflow.tflow() assert not f.response @@ -121,13 +130,12 @@ def test_authenticate(): assert not f.request.headers.get("Proxy-Authorization") f = tflow.tflow() - f.mode = "transparent" + ctx.configure(up, mode="reverse") assert not f.response up.authenticate(f) assert f.response.status_code == 401 f = tflow.tflow() - f.mode = "transparent" f.request.headers["Authorization"] = proxyauth.mkauth( "test", "test" ) @@ -139,7 +147,7 @@ def test_authenticate(): def test_handlers(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True) + ctx.configure(up, auth_nonanonymous=True, mode="regular") f = tflow.tflow() assert not f.response @@ -151,3 +159,15 @@ def test_handlers(): assert not f.response up.http_connect(f) assert f.response.status_code == 407 + + f = tflow.tflow() + f.request.method = "CONNECT" + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test" + ) + up.http_connect(f) + assert not f.response + + f2 = tflow.tflow(client_conn=f.client_conn) + up.requestheaders(f2) + assert not f2.response diff --git a/test/mitmproxy/console/__init__.py b/test/mitmproxy/contentviews/image/__init__.py index e69de29b..e69de29b 100644 --- a/test/mitmproxy/console/__init__.py +++ b/test/mitmproxy/contentviews/image/__init__.py diff --git a/test/mitmproxy/contentviews/test_image_parser.py b/test/mitmproxy/contentviews/image/test_image_parser.py index 3c8bfdf7..3cb44ca6 100644 --- a/test/mitmproxy/contentviews/test_image_parser.py +++ b/test/mitmproxy/contentviews/image/test_image_parser.py @@ -80,7 +80,7 @@ def test_parse_png(filename, metadata): # check comment "mitmproxy/data/image_parser/hopper.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '128 x 128 px'), ('background', '0'), ('comment', "b'File written by Adobe Photoshop\\xa8 4.0'") @@ -88,7 +88,7 @@ def test_parse_png(filename, metadata): # check background "mitmproxy/data/image_parser/chi.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '320 x 240 px'), ('background', '248'), ('comment', "b'Created with GIMP'") @@ -96,7 +96,7 @@ def test_parse_png(filename, metadata): # check working with color table "mitmproxy/data/image_parser/iss634.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '245 x 245 px'), ('background', '0') ], diff --git a/test/mitmproxy/contentviews/test_image.py b/test/mitmproxy/contentviews/image/test_view.py index e3dfb714..34f655a1 100644 --- a/test/mitmproxy/contentviews/test_image.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -1,6 +1,6 @@ from mitmproxy.contentviews import image from mitmproxy.test import tutils -from . import full_eval +from .. import full_eval def test_view_image(): @@ -9,9 +9,11 @@ def test_view_image(): "mitmproxy/data/image.png", "mitmproxy/data/image.gif", "mitmproxy/data/all.jpeg", - "mitmproxy/data/image.ico" + # https://bugs.python.org/issue21574 + # "mitmproxy/data/image.ico", ]: with open(tutils.test_data.path(img), "rb") as f: - assert v(f.read()) + viewname, lines = v(f.read()) + assert img.split(".")[-1].upper() in viewname - assert not v(b"flibble") + assert v(b"flibble") == ('Unknown Image', [[('header', 'Image Format: '), ('text', 'unknown')]]) diff --git a/test/mitmproxy/contentviews/test_auto.py b/test/mitmproxy/contentviews/test_auto.py index a077affa..2ff43139 100644 --- a/test/mitmproxy/contentviews/test_auto.py +++ b/test/mitmproxy/contentviews/test_auto.py @@ -30,6 +30,18 @@ def test_view_auto(): ) assert f[0].startswith("XML") + f = v( + b"<svg></svg>", + headers=http.Headers(content_type="image/svg+xml") + ) + assert f[0].startswith("XML") + + f = v( + b"verybinary", + headers=http.Headers(content_type="image/new-magic-image-format") + ) + assert f[0] == "Unknown Image" + f = v(b"\xFF" * 30) assert f[0] == "Hex" diff --git a/test/mitmproxy/contentviews/test_base.py b/test/mitmproxy/contentviews/test_base.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/contentviews/test_base.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index 1224b8db..31e382ec 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -1,12 +1,50 @@ +from unittest import mock +import pytest + from mitmproxy.contentviews import protobuf from mitmproxy.test import tutils from . import full_eval -if protobuf.ViewProtobuf.is_available(): - def test_view_protobuf_request(): - v = full_eval(protobuf.ViewProtobuf()) - p = tutils.test_data.path("mitmproxy/data/protobuf01") - content_type, output = v(open(p, "rb").read()) - assert content_type == "Protobuf" - assert output.next()[0][1] == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"' +def test_view_protobuf_request(): + v = full_eval(protobuf.ViewProtobuf()) + p = tutils.test_data.path("mitmproxy/data/protobuf01") + + with mock.patch('mitmproxy.contentviews.protobuf.ViewProtobuf.is_available'): + with mock.patch('subprocess.Popen') as n: + m = mock.Mock() + attrs = {'communicate.return_value': (b'1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"', True)} + m.configure_mock(**attrs) + n.return_value = m + + content_type, output = v(open(p, "rb").read()) + assert content_type == "Protobuf" + assert output[0] == [('text', b'1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"')] + + m.communicate = mock.MagicMock() + m.communicate.return_value = (None, None) + with pytest.raises(ValueError, matches="Failed to parse input."): + v(b'foobar') + + +def test_view_protobuf_availability(): + with mock.patch('subprocess.Popen') as n: + m = mock.Mock() + attrs = {'communicate.return_value': (b'libprotoc fake version', True)} + m.configure_mock(**attrs) + n.return_value = m + assert protobuf.ViewProtobuf().is_available() + + m = mock.Mock() + attrs = {'communicate.return_value': (b'command not found', True)} + m.configure_mock(**attrs) + n.return_value = m + assert not protobuf.ViewProtobuf().is_available() + + +def test_view_protobuf_fallback(): + with mock.patch('subprocess.Popen.communicate') as m: + m.side_effect = OSError() + v = full_eval(protobuf.ViewProtobuf()) + with pytest.raises(NotImplementedError, matches='protoc not found'): + v(b'foobar') diff --git a/test/mitmproxy/contentviews/test_wbxml.py b/test/mitmproxy/contentviews/test_wbxml.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/contentviews/test_wbxml.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_contrib_tnetstring.py b/test/mitmproxy/contrib/test_tnetstring.py index 05c4a7c9..05c4a7c9 100644 --- a/test/mitmproxy/test_contrib_tnetstring.py +++ b/test/mitmproxy/contrib/test_tnetstring.py diff --git a/test/mitmproxy/mock_urwid.py b/test/mitmproxy/mock_urwid.py deleted file mode 100644 index 9cc41abc..00000000 --- a/test/mitmproxy/mock_urwid.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys -from unittest import mock - -if os.name == "nt": - m = mock.Mock() - m.__version__ = "1.1.1" - m.Widget = mock.Mock - m.WidgetWrap = mock.Mock - sys.modules['urwid'] = m - sys.modules['urwid.util'] = mock.Mock() diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 6fe57010..90ec31fe 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -97,7 +97,7 @@ class TestRequestCore: request.host = d assert request.data.host == b"foo\xFF\x00bar" - def test_host_header_update(self): + def test_host_update_also_updates_header(self): request = treq() assert "host" not in request.headers request.host = "example.com" @@ -107,6 +107,51 @@ class TestRequestCore: request.host = "example.org" assert request.headers["Host"] == "example.org" + def test_get_host_header(self): + no_hdr = treq() + assert no_hdr.host_header is None + + h1 = treq(headers=( + (b"host", b"example.com"), + )) + assert h1.host_header == "example.com" + + h2 = treq(headers=( + (b":authority", b"example.org"), + )) + assert h2.host_header == "example.org" + + both_hdrs = treq(headers=( + (b"host", b"example.org"), + (b":authority", b"example.com"), + )) + assert both_hdrs.host_header == "example.com" + + def test_modify_host_header(self): + h1 = treq() + assert "host" not in h1.headers + assert ":authority" not in h1.headers + h1.host_header = "example.com" + assert "host" in h1.headers + assert ":authority" not in h1.headers + h1.host_header = None + assert "host" not in h1.headers + + h2 = treq(http_version=b"HTTP/2.0") + h2.host_header = "example.org" + assert "host" not in h2.headers + assert ":authority" in h2.headers + del h2.host_header + assert ":authority" not in h2.headers + + both_hdrs = treq(headers=( + (b":authority", b"example.com"), + (b"host", b"example.org"), + )) + both_hdrs.host_header = "foo.example.com" + assert both_hdrs.headers["Host"] == "foo.example.com" + assert both_hdrs.headers[":authority"] == "foo.example.com" + class TestRequestUtils: """ diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index 11ab1b81..2064aab8 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -85,6 +85,26 @@ surrogates_quoted = ( ) +def test_empty_key_trailing_equal_sign(): + """ + Some HTTP clients don't send trailing equal signs for parameters without assigned value, e.g. they send + foo=bar&baz&qux=quux + instead of + foo=bar&baz=&qux=quux + The respective behavior of encode() should be driven by a reference string given in similar_to parameter + """ + reference_without_equal = "key1=val1&key2&key3=val3" + reference_with_equal = "key1=val1&key2=&key3=val3" + + post_data_empty_key_middle = [('one', 'two'), ('emptykey', ''), ('three', 'four')] + post_data_empty_key_end = [('one', 'two'), ('three', 'four'), ('emptykey', '')] + + assert url.encode(post_data_empty_key_middle, similar_to = reference_with_equal) == "one=two&emptykey=&three=four" + assert url.encode(post_data_empty_key_end, similar_to = reference_with_equal) == "one=two&three=four&emptykey=" + assert url.encode(post_data_empty_key_middle, similar_to = reference_without_equal) == "one=two&emptykey&three=four" + assert url.encode(post_data_empty_key_end, similar_to = reference_without_equal) == "one=two&three=four&emptykey" + + def test_encode(): assert url.encode([('foo', 'bar')]) assert url.encode([('foo', surrogates)]) diff --git a/test/mitmproxy/test_platform_pf.py b/test/mitmproxy/platform/test_pf.py index f644bcc5..f644bcc5 100644 --- a/test/mitmproxy/test_platform_pf.py +++ b/test/mitmproxy/platform/test_pf.py diff --git a/test/mitmproxy/protocol/__init__.py b/test/mitmproxy/proxy/__init__.py index e69de29b..e69de29b 100644 --- a/test/mitmproxy/protocol/__init__.py +++ b/test/mitmproxy/proxy/__init__.py diff --git a/test/mitmproxy/proxy/modes/test_http_proxy.py b/test/mitmproxy/proxy/modes/test_http_proxy.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/modes/test_http_proxy.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_reverse_proxy.py b/test/mitmproxy/proxy/modes/test_reverse_proxy.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/modes/test_reverse_proxy.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_socks_proxy.py b/test/mitmproxy/proxy/modes/test_socks_proxy.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/modes/test_socks_proxy.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_transparent_proxy.py b/test/mitmproxy/proxy/modes/test_transparent_proxy.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/modes/test_transparent_proxy.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/__init__.py b/test/mitmproxy/proxy/protocol/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/proxy/protocol/__init__.py diff --git a/test/mitmproxy/proxy/protocol/test_base.py b/test/mitmproxy/proxy/protocol/test_base.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/protocol/test_base.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/test_http.py b/test/mitmproxy/proxy/protocol/test_http.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/protocol/test_http.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/protocol/test_http1.py b/test/mitmproxy/proxy/protocol/test_http1.py index 44a9effa..07cd7dcc 100644 --- a/test/mitmproxy/protocol/test_http1.py +++ b/test/mitmproxy/proxy/protocol/test_http1.py @@ -2,7 +2,7 @@ from mitmproxy.test import tflow from mitmproxy.net.http import http1 from mitmproxy.net.tcp import TCPClient from mitmproxy.test.tutils import treq -from .. import tservers +from ... import tservers class TestHTTPFlow: diff --git a/test/mitmproxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 8e8ba644..cb9c0474 100644 --- a/test/mitmproxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -4,27 +4,25 @@ import os import tempfile import traceback - +import pytest import h2 from mitmproxy import options from mitmproxy.proxy.config import ProxyConfig import mitmproxy.net -from ...mitmproxy.net import tservers as net_tservers +from ....mitmproxy.net import tservers as net_tservers from mitmproxy import exceptions from mitmproxy.net.http import http1, http2 -from .. import tservers -from ...conftest import requires_alpn +from ... import tservers +from ....conftest import requires_alpn import logging logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) logging.getLogger("passlib.utils.compat").setLevel(logging.WARNING) logging.getLogger("passlib.registry").setLevel(logging.WARNING) -logging.getLogger("PIL.Image").setLevel(logging.WARNING) -logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) # inspect the log: @@ -274,13 +272,13 @@ class TestSimple(_Http2Test): @requires_alpn -class TestRequestWithPriority(_Http2Test): +class TestForbiddenHeaders(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - elif isinstance(event, h2.events.RequestReceived): + elif isinstance(event, h2.events.StreamEnded): import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -291,60 +289,18 @@ class TestRequestWithPriority(_Http2Test): warnings.simplefilter("ignore") - headers = [(':status', '200')] - if event.priority_updated: - headers.append(('priority_exclusive', str(event.priority_updated.exclusive).encode())) - headers.append(('priority_depends_on', str(event.priority_updated.depends_on).encode())) - headers.append(('priority_weight', str(event.priority_updated.weight).encode())) - h2_conn.send_headers(event.stream_id, headers) + h2_conn.config.validate_outbound_headers = False + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('keep-alive', 'foobar'), + ]) + h2_conn.send_data(event.stream_id, b'response body') h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_request_with_priority(self): - client, h2_conn = self._setup_connection() - - self._send_request( - client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address.port)), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - priority_exclusive=True, - priority_depends_on=42424242, - priority_weight=42, - ) - - done = False - while not done: - try: - raw = b''.join(http2.read_raw_frame(client.rfile)) - events = h2_conn.receive_data(raw) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response.headers['priority_exclusive'] == 'True' - assert self.master.state.flows[0].response.headers['priority_depends_on'] == '42424242' - assert self.master.state.flows[0].response.headers['priority_weight'] == '42' - - def test_request_without_priority(self): + def test_forbidden_headers(self): client, h2_conn = self._setup_connection() self._send_request( @@ -355,8 +311,7 @@ class TestRequestWithPriority(_Http2Test): (':method', 'GET'), (':scheme', 'https'), (':path', '/'), - ], - ) + ]) done = False while not done: @@ -371,7 +326,9 @@ class TestRequestWithPriority(_Http2Test): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.ResponseReceived): + assert 'keep-alive' not in event.headers + elif isinstance(event, h2.events.StreamEnded): done = True h2_conn.close_connection() @@ -379,21 +336,17 @@ class TestRequestWithPriority(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert 'priority_exclusive' not in self.master.state.flows[0].response.headers - assert 'priority_depends_on' not in self.master.state.flows[0].response.headers - assert 'priority_weight' not in self.master.state.flows[0].response.headers + assert self.master.state.flows[0].response.status_code == 200 + assert self.master.state.flows[0].response.headers['keep-alive'] == 'foobar' @requires_alpn -class TestPriority(_Http2Test): - priority_data = None +class TestRequestWithPriority(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - elif isinstance(event, h2.events.PriorityUpdated): - cls.priority_data = (event.exclusive, event.depends_on, event.weight) elif isinstance(event, h2.events.RequestReceived): import warnings with warnings.catch_warnings(): @@ -406,18 +359,26 @@ class TestPriority(_Http2Test): warnings.simplefilter("ignore") headers = [(':status', '200')] + if event.priority_updated: + headers.append(('priority_exclusive', str(event.priority_updated.exclusive).encode())) + headers.append(('priority_depends_on', str(event.priority_updated.depends_on).encode())) + headers.append(('priority_weight', str(event.priority_updated.weight).encode())) h2_conn.send_headers(event.stream_id, headers) h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_priority(self): - client, h2_conn = self._setup_connection() + @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ + (True, (True, 42424242, 42), ('True', '42424242', '42')), + (False, (True, 42424242, 42), (None, None, None)), + (True, (None, None, None), (None, None, None)), + (False, (None, None, None), (None, None, None)), + ]) + def test_request_with_priority(self, http2_priority_enabled, priority, expected_priority): + self.config.options.http2_priority = http2_priority_enabled - h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() + client, h2_conn = self._setup_connection() self._send_request( client.wfile, @@ -428,6 +389,9 @@ class TestPriority(_Http2Test): (':scheme', 'https'), (':path', '/'), ], + priority_exclusive=priority[0], + priority_depends_on=priority[1], + priority_weight=priority[2], ) done = False @@ -451,12 +415,15 @@ class TestPriority(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert self.priority_data == (True, 0, 42) + + resp = self.master.state.flows[0].response + assert resp.headers.get('priority_exclusive', None) == expected_priority[0] + assert resp.headers.get('priority_depends_on', None) == expected_priority[1] + assert resp.headers.get('priority_weight', None) == expected_priority[2] @requires_alpn -class TestPriorityWithExistingStream(_Http2Test): - priority_data = [] +class TestPriority(_Http2Test): @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): @@ -465,8 +432,6 @@ class TestPriorityWithExistingStream(_Http2Test): elif isinstance(event, h2.events.PriorityUpdated): cls.priority_data.append((event.exclusive, event.depends_on, event.weight)) elif isinstance(event, h2.events.RequestReceived): - assert not event.priority_updated - import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -479,17 +444,27 @@ class TestPriorityWithExistingStream(_Http2Test): headers = [(':status', '200')] h2_conn.send_headers(event.stream_id, headers) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - elif isinstance(event, h2.events.StreamEnded): h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_priority_with_existing_stream(self): + @pytest.mark.parametrize("prioritize_before", [True, False]) + @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ + (True, (True, 42424242, 42), [(True, 42424242, 42)]), + (False, (True, 42424242, 42), []), + ]) + def test_priority(self, prioritize_before, http2_priority_enabled, priority, expected_priority): + self.config.options.http2_priority = http2_priority_enabled + self.__class__.priority_data = [] + client, h2_conn = self._setup_connection() + if prioritize_before: + h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + self._send_request( client.wfile, h2_conn, @@ -499,13 +474,14 @@ class TestPriorityWithExistingStream(_Http2Test): (':scheme', 'https'), (':path', '/'), ], - end_stream=False, + end_stream=prioritize_before, ) - h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) - h2_conn.end_stream(1) - client.wfile.write(h2_conn.data_to_send()) - client.wfile.flush() + if not prioritize_before: + h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) + h2_conn.end_stream(1) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() done = False while not done: @@ -528,7 +504,7 @@ class TestPriorityWithExistingStream(_Http2Test): client.wfile.flush() assert len(self.master.state.flows) == 1 - assert self.priority_data == [(True, 0, 42)] + assert self.priority_data == expected_priority @requires_alpn diff --git a/test/mitmproxy/proxy/protocol/test_http_replay.py b/test/mitmproxy/proxy/protocol/test_http_replay.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/protocol/test_http_replay.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/test_rawtcp.py b/test/mitmproxy/proxy/protocol/test_rawtcp.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/protocol/test_rawtcp.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/protocol/test_tls.py b/test/mitmproxy/proxy/protocol/test_tls.py index e17ee46f..e17ee46f 100644 --- a/test/mitmproxy/protocol/test_tls.py +++ b/test/mitmproxy/proxy/protocol/test_tls.py diff --git a/test/mitmproxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 73ee8b35..4ea01d34 100644 --- a/test/mitmproxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -9,17 +9,17 @@ from mitmproxy.http import HTTPFlow from mitmproxy.websocket import WebSocketFlow from mitmproxy.proxy.config import ProxyConfig -import mitmproxy.net +from mitmproxy.net import tcp from mitmproxy.net import http -from ...mitmproxy.net import tservers as net_tservers -from .. import tservers +from ....mitmproxy.net import tservers as net_tservers +from ... import tservers from mitmproxy.net import websockets class _WebSocketServerBase(net_tservers.ServerTestBase): - class handler(mitmproxy.net.tcp.BaseHandler): + class handler(tcp.BaseHandler): def handle(self): try: @@ -80,7 +80,7 @@ class _WebSocketTestBase: self.server.server.handle_websockets = self.handle_websockets def _setup_connection(self): - client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) + client = tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() request = http.Request( diff --git a/test/mitmproxy/test_proxy_config.py b/test/mitmproxy/proxy/test_config.py index 4272d952..4272d952 100644 --- a/test/mitmproxy/test_proxy_config.py +++ b/test/mitmproxy/proxy/test_config.py diff --git a/test/mitmproxy/proxy/test_root_context.py b/test/mitmproxy/proxy/test_root_context.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/proxy/test_root_context.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 9cd47cea..0be772a4 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -21,8 +21,8 @@ from mitmproxy.net.tcp import Address from pathod import pathoc from pathod import pathod -from . import tservers -from ..conftest import skip_appveyor +from .. import tservers +from ...conftest import skip_appveyor """ diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_connections.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_ctx.py b/test/mitmproxy/test_ctx.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_ctx.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_exceptions.py b/test/mitmproxy/test_exceptions.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_exceptions.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_export.py index 457d8836..457d8836 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_export.py diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 65e6845f..a78e5f80 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -10,7 +10,6 @@ from mitmproxy.exceptions import FlowReadException, Kill from mitmproxy import flow from mitmproxy import http from mitmproxy import connections -from mitmproxy import tcp from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer from mitmproxy import master @@ -157,117 +156,6 @@ class TestHTTPFlow: assert f.response.raw_content == b"abarb" -class TestWebSocketFlow: - - def test_copy(self): - f = tflow.twebsocketflow() - f.get_state() - f2 = f.copy() - a = f.get_state() - b = f2.get_state() - del a["id"] - del b["id"] - del a["handshake_flow"]["id"] - del b["handshake_flow"]["id"] - assert a == b - assert not f == f2 - assert f is not f2 - - assert f.client_key == f2.client_key - assert f.client_protocol == f2.client_protocol - assert f.client_extensions == f2.client_extensions - assert f.server_accept == f2.server_accept - assert f.server_protocol == f2.server_protocol - assert f.server_extensions == f2.server_extensions - assert f.messages is not f2.messages - assert f.handshake_flow is not f2.handshake_flow - - for m in f.messages: - m2 = m.copy() - m2.set_state(m2.get_state()) - assert m is not m2 - assert m.get_state() == m2.get_state() - - f = tflow.twebsocketflow(err=True) - f2 = f.copy() - assert f is not f2 - assert f.handshake_flow is not f2.handshake_flow - assert f.error.get_state() == f2.error.get_state() - assert f.error is not f2.error - - def test_match(self): - f = tflow.twebsocketflow() - assert not flowfilter.match("~b nonexistent", f) - assert flowfilter.match(None, f) - assert not flowfilter.match("~b nonexistent", f) - - f = tflow.twebsocketflow(err=True) - assert flowfilter.match("~e", f) - - with pytest.raises(ValueError): - flowfilter.match("~", f) - - def test_repr(self): - f = tflow.twebsocketflow() - assert 'WebSocketFlow' in repr(f) - assert 'binary message: ' in repr(f.messages[0]) - assert 'text message: ' in repr(f.messages[1]) - - -class TestTCPFlow: - - def test_copy(self): - f = tflow.ttcpflow() - f.get_state() - f2 = f.copy() - a = f.get_state() - b = f2.get_state() - del a["id"] - del b["id"] - assert a == b - assert not f == f2 - assert f is not f2 - - assert f.messages is not f2.messages - - for m in f.messages: - assert m.get_state() - m2 = m.copy() - assert not m == m2 - assert m is not m2 - - a = m.get_state() - b = m2.get_state() - assert a == b - - m = tcp.TCPMessage(False, 'foo') - m.set_state(f.messages[0].get_state()) - assert m.timestamp == f.messages[0].timestamp - - f = tflow.ttcpflow(err=True) - f2 = f.copy() - assert f is not f2 - assert f.error.get_state() == f2.error.get_state() - assert f.error is not f2.error - - def test_match(self): - f = tflow.ttcpflow() - assert not flowfilter.match("~b nonexistent", f) - assert flowfilter.match(None, f) - assert not flowfilter.match("~b nonexistent", f) - - f = tflow.ttcpflow(err=True) - assert flowfilter.match("~e", f) - - with pytest.raises(ValueError): - flowfilter.match("~", f) - - def test_repr(self): - f = tflow.ttcpflow() - assert 'TCPFlow' in repr(f) - assert '-> ' in repr(f.messages[0]) - - class TestSerialize: def _treader(self): diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_http.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_io.py b/test/mitmproxy/test_io.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_io.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_io_compat.py index 288de4fc..288de4fc 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_io_compat.py diff --git a/test/mitmproxy/test_log.py b/test/mitmproxy/test_log.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_log.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_master.py b/test/mitmproxy/test_master.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_master.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/test_options.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/test_tcp.py b/test/mitmproxy/test_tcp.py new file mode 100644 index 00000000..dce6493c --- /dev/null +++ b/test/mitmproxy/test_tcp.py @@ -0,0 +1,59 @@ +import pytest + +from mitmproxy import tcp +from mitmproxy import flowfilter +from mitmproxy.test import tflow + + +class TestTCPFlow: + + def test_copy(self): + f = tflow.ttcpflow() + f.get_state() + f2 = f.copy() + a = f.get_state() + b = f2.get_state() + del a["id"] + del b["id"] + assert a == b + assert not f == f2 + assert f is not f2 + + assert f.messages is not f2.messages + + for m in f.messages: + assert m.get_state() + m2 = m.copy() + assert not m == m2 + assert m is not m2 + + a = m.get_state() + b = m2.get_state() + assert a == b + + m = tcp.TCPMessage(False, 'foo') + m.set_state(f.messages[0].get_state()) + assert m.timestamp == f.messages[0].timestamp + + f = tflow.ttcpflow(err=True) + f2 = f.copy() + assert f is not f2 + assert f.error.get_state() == f2.error.get_state() + assert f.error is not f2.error + + def test_match(self): + f = tflow.ttcpflow() + assert not flowfilter.match("~b nonexistent", f) + assert flowfilter.match(None, f) + assert not flowfilter.match("~b nonexistent", f) + + f = tflow.ttcpflow(err=True) + assert flowfilter.match("~e", f) + + with pytest.raises(ValueError): + flowfilter.match("~", f) + + def test_repr(self): + f = tflow.ttcpflow() + assert 'TCPFlow' in repr(f) + assert '-> ' in repr(f.messages[0]) diff --git a/test/mitmproxy/test_websocket.py b/test/mitmproxy/test_websocket.py new file mode 100644 index 00000000..f2963390 --- /dev/null +++ b/test/mitmproxy/test_websocket.py @@ -0,0 +1,62 @@ +import pytest + +from mitmproxy import flowfilter +from mitmproxy.test import tflow + + +class TestWebSocketFlow: + + def test_copy(self): + f = tflow.twebsocketflow() + f.get_state() + f2 = f.copy() + a = f.get_state() + b = f2.get_state() + del a["id"] + del b["id"] + del a["handshake_flow"]["id"] + del b["handshake_flow"]["id"] + assert a == b + assert not f == f2 + assert f is not f2 + + assert f.client_key == f2.client_key + assert f.client_protocol == f2.client_protocol + assert f.client_extensions == f2.client_extensions + assert f.server_accept == f2.server_accept + assert f.server_protocol == f2.server_protocol + assert f.server_extensions == f2.server_extensions + assert f.messages is not f2.messages + assert f.handshake_flow is not f2.handshake_flow + + for m in f.messages: + m2 = m.copy() + m2.set_state(m2.get_state()) + assert m is not m2 + assert m.get_state() == m2.get_state() + + f = tflow.twebsocketflow(err=True) + f2 = f.copy() + assert f is not f2 + assert f.handshake_flow is not f2.handshake_flow + assert f.error.get_state() == f2.error.get_state() + assert f.error is not f2.error + + def test_match(self): + f = tflow.twebsocketflow() + assert not flowfilter.match("~b nonexistent", f) + assert flowfilter.match(None, f) + assert not flowfilter.match("~b nonexistent", f) + + f = tflow.twebsocketflow(err=True) + assert flowfilter.match("~e", f) + + with pytest.raises(ValueError): + flowfilter.match("~", f) + + def test_repr(self): + f = tflow.twebsocketflow() + assert f.message_info(f.messages[0]) + assert 'WebSocketFlow' in repr(f) + assert 'binary message: ' in repr(f.messages[0]) + assert 'text message: ' in repr(f.messages[1]) diff --git a/test/mitmproxy/tools/__init__.py b/test/mitmproxy/tools/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/tools/__init__.py diff --git a/test/mitmproxy/tools/console/__init__.py b/test/mitmproxy/tools/console/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/tools/console/__init__.py diff --git a/test/mitmproxy/console/test_common.py b/test/mitmproxy/tools/console/test_common.py index 236ebb41..3ab4fd67 100644 --- a/test/mitmproxy/console/test_common.py +++ b/test/mitmproxy/tools/console/test_common.py @@ -1,7 +1,7 @@ from mitmproxy.test import tflow from mitmproxy.tools.console import common -from ...conftest import skip_appveyor +from ....conftest import skip_appveyor @skip_appveyor diff --git a/test/mitmproxy/console/test_help.py b/test/mitmproxy/tools/console/test_help.py index 86b842d8..ac3011e6 100644 --- a/test/mitmproxy/console/test_help.py +++ b/test/mitmproxy/tools/console/test_help.py @@ -1,6 +1,6 @@ import mitmproxy.tools.console.help as help -from ...conftest import skip_appveyor +from ....conftest import skip_appveyor @skip_appveyor diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 1c89467c..0bf3734b 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -1,10 +1,10 @@ from mitmproxy.test import tflow -import mitmproxy.test.tutils +from mitmproxy.test import tutils from mitmproxy.tools import console from mitmproxy import proxy from mitmproxy import options from mitmproxy.tools.console import common -from .. import tservers +from ... import tservers def test_format_keyvals(): @@ -45,12 +45,12 @@ class TestMaster(tservers.MasterTest): def test_intercept(self): """regression test for https://github.com/mitmproxy/mitmproxy/issues/1605""" m = self.mkmaster(intercept="~b bar") - f = tflow.tflow(req=mitmproxy.test.tutils.treq(content=b"foo")) + f = tflow.tflow(req=tutils.treq(content=b"foo")) m.request(f) assert not m.view[0].intercepted - f = tflow.tflow(req=mitmproxy.test.tutils.treq(content=b"bar")) + f = tflow.tflow(req=tutils.treq(content=b"bar")) m.request(f) assert m.view[1].intercepted - f = tflow.tflow(resp=mitmproxy.test.tutils.tresp(content=b"bar")) + f = tflow.tflow(resp=tutils.tresp(content=b"bar")) m.request(f) assert m.view[2].intercepted diff --git a/test/mitmproxy/console/test_palettes.py b/test/mitmproxy/tools/console/test_palettes.py index 3892797d..1c7e1df1 100644 --- a/test/mitmproxy/console/test_palettes.py +++ b/test/mitmproxy/tools/console/test_palettes.py @@ -1,6 +1,6 @@ import mitmproxy.tools.console.palettes as palettes -from ...conftest import skip_appveyor +from ....conftest import skip_appveyor @skip_appveyor diff --git a/test/mitmproxy/console/test_pathedit.py b/test/mitmproxy/tools/console/test_pathedit.py index bd064e5f..bd064e5f 100644 --- a/test/mitmproxy/console/test_pathedit.py +++ b/test/mitmproxy/tools/console/test_pathedit.py diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py index 96d5ae31..96d5ae31 100644 --- a/test/mitmproxy/test_cmdline.py +++ b/test/mitmproxy/tools/test_cmdline.py diff --git a/test/mitmproxy/test_tools_dump.py b/test/mitmproxy/tools/test_dump.py index f8a88871..b4183725 100644 --- a/test/mitmproxy/test_tools_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -8,7 +8,7 @@ from mitmproxy import controller from mitmproxy.tools import dump from mitmproxy.test import tutils -from . import tservers +from .. import tservers class TestDumpMaster(tservers.MasterTest): diff --git a/test/mitmproxy/tools/web/__init__.py b/test/mitmproxy/tools/web/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/tools/web/__init__.py diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/tools/web/test_app.py index 00dc2c7c..00dc2c7c 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/tools/web/test_app.py diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/tools/web/test_master.py index d4190ffb..27f99a18 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/tools/web/test_master.py @@ -3,7 +3,7 @@ from mitmproxy import proxy from mitmproxy import options from mitmproxy.proxy.config import ProxyConfig -from . import tservers +from ... import tservers class TestWebMaster(tservers.MasterTest): diff --git a/test/mitmproxy/utils/test_bits.py b/test/mitmproxy/utils/test_bits.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/mitmproxy/utils/test_bits.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/mitmproxy/utils/test_debug.py b/test/mitmproxy/utils/test_debug.py index 22f8dc66..4371ef70 100644 --- a/test/mitmproxy/utils/test_debug.py +++ b/test/mitmproxy/utils/test_debug.py @@ -1,13 +1,20 @@ import io import subprocess +import sys from unittest import mock +import pytest from mitmproxy.utils import debug -def test_dump_system_info(): - assert debug.dump_system_info() +@pytest.mark.parametrize("precompiled", [True, False]) +def test_dump_system_info_precompiled(precompiled): + sys.frozen = None + with mock.patch.object(sys, 'frozen', precompiled): + assert ("Precompiled Binary" in debug.dump_system_info()) == precompiled + +def test_dump_system_info_version(): with mock.patch('subprocess.check_output') as m: m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long') assert 'release version' in debug.dump_system_info() diff --git a/test/pathod/language/__init__.py b/test/pathod/language/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/pathod/language/__init__.py diff --git a/test/pathod/test_language_actions.py b/test/pathod/language/test_actions.py index 9740e5c7..9740e5c7 100644 --- a/test/pathod/test_language_actions.py +++ b/test/pathod/language/test_actions.py diff --git a/test/pathod/test_language_base.py b/test/pathod/language/test_base.py index 85e9e53b..85e9e53b 100644 --- a/test/pathod/test_language_base.py +++ b/test/pathod/language/test_base.py diff --git a/test/pathod/language/test_exceptions.py b/test/pathod/language/test_exceptions.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/pathod/language/test_exceptions.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/pathod/test_language_generators.py b/test/pathod/language/test_generators.py index b3ce0335..b3ce0335 100644 --- a/test/pathod/test_language_generators.py +++ b/test/pathod/language/test_generators.py diff --git a/test/pathod/test_language_http.py b/test/pathod/language/test_http.py index 6ab43fe0..a5b35c05 100644 --- a/test/pathod/test_language_http.py +++ b/test/pathod/language/test_http.py @@ -4,7 +4,7 @@ import pytest from pathod import language from pathod.language import http, base -from . import tservers +from .. import tservers def parse_request(s): diff --git a/test/pathod/test_language_http2.py b/test/pathod/language/test_http2.py index 4f89adb8..4f89adb8 100644 --- a/test/pathod/test_language_http2.py +++ b/test/pathod/language/test_http2.py diff --git a/test/pathod/language/test_message.py b/test/pathod/language/test_message.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/pathod/language/test_message.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/pathod/test_language_websocket.py b/test/pathod/language/test_websockets.py index e5046591..ed766bca 100644 --- a/test/pathod/test_language_websocket.py +++ b/test/pathod/language/test_websockets.py @@ -4,7 +4,7 @@ from pathod import language from pathod.language import websockets import mitmproxy.net.websockets -from . import tservers +from .. import tservers def parse_request(s): diff --git a/test/pathod/test_language_writer.py b/test/pathod/language/test_writer.py index 7feb985d..7feb985d 100644 --- a/test/pathod/test_language_writer.py +++ b/test/pathod/language/test_writer.py diff --git a/test/pathod/protocols/__init__.py b/test/pathod/protocols/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/pathod/protocols/__init__.py diff --git a/test/pathod/protocols/test_http.py b/test/pathod/protocols/test_http.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/pathod/protocols/test_http.py @@ -0,0 +1 @@ +# TODO: write tests diff --git a/test/pathod/test_protocols_http2.py b/test/pathod/protocols/test_http2.py index 5bb31031..1c074197 100644 --- a/test/pathod/test_protocols_http2.py +++ b/test/pathod/protocols/test_http2.py @@ -7,11 +7,11 @@ from mitmproxy.net import tcp, http from mitmproxy.net.http import http2 from mitmproxy import exceptions -from ..mitmproxy.net import tservers as net_tservers +from ...mitmproxy.net import tservers as net_tservers from pathod.protocols.http2 import HTTP2StateProtocol, TCPHandler -from ..conftest import requires_alpn +from ...conftest import requires_alpn class TestTCPHandlerWrapper: diff --git a/test/pathod/protocols/test_websockets.py b/test/pathod/protocols/test_websockets.py new file mode 100644 index 00000000..777ab4dd --- /dev/null +++ b/test/pathod/protocols/test_websockets.py @@ -0,0 +1 @@ +# TODO: write tests @@ -11,14 +11,9 @@ passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* O setenv = HOME = {envtmpdir} commands = mitmdump --version - pytest --timeout 60 --cov-report='' --cov=mitmproxy --cov=pathod \ - --full-cov=mitmproxy/ \ - --no-full-cov=mitmproxy/contentviews/__init__.py --no-full-cov=mitmproxy/contentviews/protobuf.py --no-full-cov=mitmproxy/contentviews/wbxml.py --no-full-cov=mitmproxy/contentviews/xml_html.py \ - --no-full-cov=mitmproxy/net/tcp.py --no-full-cov=mitmproxy/net/http/cookies.py --no-full-cov=mitmproxy/net/http/encoding.py --no-full-cov=mitmproxy/net/http/message.py --no-full-cov=mitmproxy/net/http/url.py \ - --no-full-cov=mitmproxy/proxy/protocol/ --no-full-cov=mitmproxy/proxy/config.py --no-full-cov=mitmproxy/proxy/root_context.py --no-full-cov=mitmproxy/proxy/server.py \ - --no-full-cov=mitmproxy/tools/ \ - --no-full-cov=mitmproxy/certs.py --no-full-cov=mitmproxy/connections.py --no-full-cov=mitmproxy/controller.py --no-full-cov=mitmproxy/export.py --no-full-cov=mitmproxy/flow.py --no-full-cov=mitmproxy/flowfilter.py --no-full-cov=mitmproxy/http.py --no-full-cov=mitmproxy/io_compat.py --no-full-cov=mitmproxy/master.py --no-full-cov=mitmproxy/optmanager.py \ - --full-cov=pathod/ --no-full-cov=pathod/pathoc.py --no-full-cov=pathod/pathod.py --no-full-cov=pathod/test.py --no-full-cov=pathod/protocols/http2.py \ + pytest --timeout 60 --cov-report='' \ + --cov=mitmproxy --cov=pathod \ + --full-cov=mitmproxy/ --full-cov=pathod/ \ {posargs} {env:CI_COMMANDS:python -c ""} @@ -29,9 +24,10 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] commands = mitmdump --version - flake8 --jobs 8 --count mitmproxy pathod examples test release + flake8 --jobs 8 mitmproxy pathod examples test release + python3 test/filename_matching.py rstcheck README.rst - mypy --silent-imports \ + mypy --ignore-missing-imports --follow-imports=skip \ mitmproxy/addons/ \ mitmproxy/addonmanager.py \ mitmproxy/proxy/protocol/ \ @@ -40,6 +36,12 @@ commands = mitmproxy/tools/web/ \ mitmproxy/contentviews/ +[testenv:individual_coverage] +deps = + -rrequirements.txt +commands = + python3 test/individual_coverage.py + [testenv:wheel] recreate = True deps = |