diff options
35 files changed, 342 insertions, 177 deletions
@@ -11,6 +11,7 @@ MANIFEST .cache/ .tox*/ build/ +dist/ mitmproxy/contrib/kaitaistruct/*.ksy # UI diff --git a/docs/transparent/osx.rst b/docs/transparent/osx.rst index 40e91fac..5d4ec612 100644 --- a/docs/transparent/osx.rst +++ b/docs/transparent/osx.rst @@ -17,8 +17,7 @@ Note that this means we don't support transparent mode for earlier versions of O .. code-block:: none - rdr on en2 inet proto tcp to any port 80 -> 127.0.0.1 port 8080 - rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080 + rdr on en0 inet proto tcp to any port {80, 443} -> 127.0.0.1 port 8080 These rules tell pf to redirect all traffic destined for port 80 or 443 to the local mitmproxy instance running on port 8080. You should diff --git a/examples/complex/change_upstream_proxy.py b/examples/complex/change_upstream_proxy.py index 49d5379f..089a9df5 100644 --- a/examples/complex/change_upstream_proxy.py +++ b/examples/complex/change_upstream_proxy.py @@ -1,3 +1,6 @@ +from mitmproxy import http +import typing + # This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy # in upstream proxy mode. # @@ -6,7 +9,7 @@ # If you want to change the target server, you should modify flow.request.host and flow.request.port -def proxy_address(flow): +def proxy_address(flow: http.HTTPFlow) -> typing.Tuple[str, int]: # Poor man's loadbalancing: route every second domain through the alternative proxy. if hash(flow.request.host) % 2 == 1: return ("localhost", 8082) @@ -14,7 +17,7 @@ def proxy_address(flow): return ("localhost", 8081) -def request(flow): +def request(flow: http.HTTPFlow) -> None: if flow.request.method == "CONNECT": # If the decision is done by domain, one could also modify the server address here. # We do it after CONNECT here to have the request data available as well. diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 66a81a7d..9e287a19 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -7,22 +7,24 @@ import json import base64 import zlib import os +import typing # noqa from datetime import datetime from datetime import timezone import mitmproxy +from mitmproxy import connections # noqa from mitmproxy import version from mitmproxy import ctx from mitmproxy.utils import strutils from mitmproxy.net.http import cookies -HAR = {} +HAR = {} # type: typing.Dict # A list of server seen till now is maintained so we can avoid # using 'connect' time for entries that use an existing connection. -SERVERS_SEEN = set() +SERVERS_SEEN = set() # type: typing.Set[connections.ServerConnection] def load(l): diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index 2f60c8b9..c3f8c4f7 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -3,13 +3,16 @@ This script implements an sslstrip-like attack based on mitmproxy. https://moxie.org/software/sslstrip/ """ import re -import urllib +import urllib.parse +import typing # noqa + +from mitmproxy import http # set of SSL/TLS capable hosts -secure_hosts = set() +secure_hosts = set() # type: typing.Set[str] -def request(flow): +def request(flow: http.HTTPFlow) -> None: flow.request.headers.pop('If-Modified-Since', None) flow.request.headers.pop('Cache-Control', None) @@ -27,7 +30,7 @@ def request(flow): flow.request.host = flow.request.pretty_host -def response(flow): +def response(flow: http.HTTPFlow) -> None: flow.response.headers.pop('Strict-Transport-Security', None) flow.response.headers.pop('Public-Key-Pins', None) diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index 4b35c6c1..0ee38cd4 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -35,14 +35,17 @@ Line: 1029zxcs'd"ao<ac>so[sb]po(pc)se;sl/bsl\eq=3847asd """ -from mitmproxy import ctx +from html.parser import HTMLParser +from typing import Dict, Union, Tuple, Optional, List, NamedTuple from socket import gaierror, gethostbyname from urllib.parse import urlparse -import requests import re -from html.parser import HTMLParser + +import requests + from mitmproxy import http -from typing import Dict, Union, Tuple, Optional, List, NamedTuple +from mitmproxy import ctx + # The actual payload is put between a frontWall and a backWall to make it easy # to locate the payload with regular expressions @@ -83,15 +86,16 @@ def get_cookies(flow: http.HTTPFlow) -> Cookies: return {name: value for name, value in flow.request.cookies.fields} -def find_unclaimed_URLs(body: Union[str, bytes], requestUrl: bytes) -> None: +def find_unclaimed_URLs(body: str, requestUrl: bytes) -> None: """ Look for unclaimed URLs in script tags and log them if found""" - def getValue(attrs: List[Tuple[str, str]], attrName: str) -> str: + def getValue(attrs: List[Tuple[str, str]], attrName: str) -> Optional[str]: for name, value in attrs: if attrName == name: return value + return None class ScriptURLExtractor(HTMLParser): - script_URLs = [] + script_URLs = [] # type: List[str] def handle_starttag(self, tag, attrs): if (tag == "script" or tag == "iframe") and "src" in [name for name, value in attrs]: @@ -100,13 +104,10 @@ def find_unclaimed_URLs(body: Union[str, bytes], requestUrl: bytes) -> None: self.script_URLs.append(getValue(attrs, "href")) parser = ScriptURLExtractor() - try: - parser.feed(body) - except TypeError: - parser.feed(body.decode('utf-8')) + parser.feed(body) for url in parser.script_URLs: - parser = urlparse(url) - domain = parser.netloc + url_parser = urlparse(url) + domain = url_parser.netloc try: gethostbyname(domain) except gaierror: @@ -178,10 +179,11 @@ def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None: if not sqli_info: return ctx.log.error("===== SQLi Found =====") - ctx.log.error("SQLi URL: %s" % sqli_info.url.decode('utf-8')) - ctx.log.error("Injection Point: %s" % sqli_info.injection_point.decode('utf-8')) - ctx.log.error("Regex used: %s" % sqli_info.regex.decode('utf-8')) - ctx.log.error("Suspected DBMS: %s" % sqli_info.dbms.decode('utf-8')) + ctx.log.error("SQLi URL: %s" % sqli_info.url) + ctx.log.error("Injection Point: %s" % sqli_info.injection_point) + ctx.log.error("Regex used: %s" % sqli_info.regex) + ctx.log.error("Suspected DBMS: %s" % sqli_info.dbms) + return def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection_point: str) -> Optional[SQLiData]: @@ -202,20 +204,21 @@ def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection "Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message", r"Sybase.*Server message.*"), } for dbms, regexes in DBMS_ERRORS.items(): - for regex in regexes: + for regex in regexes: # type: ignore if re.search(regex, new_body, re.IGNORECASE) and not re.search(regex, original_body, re.IGNORECASE): return SQLiData(request_URL, injection_point, regex, dbms) + return None # A qc is either ' or " -def inside_quote(qc: str, substring: bytes, text_index: int, body: bytes) -> bool: +def inside_quote(qc: str, substring_bytes: bytes, text_index: int, body_bytes: bytes) -> bool: """ Whether the Numberth occurence of the first string in the second string is inside quotes as defined by the supplied QuoteChar """ - substring = substring.decode('utf-8') - body = body.decode('utf-8') + substring = substring_bytes.decode('utf-8') + body = body_bytes.decode('utf-8') num_substrings_found = 0 in_quote = False for index, char in enumerate(body): @@ -238,20 +241,20 @@ def inside_quote(qc: str, substring: bytes, text_index: int, body: bytes) -> boo return False -def paths_to_text(html: str, str: str) -> List[str]: +def paths_to_text(html: str, string: str) -> List[str]: """ Return list of Paths to a given str in the given HTML tree - Note that it does a BFS """ - def remove_last_occurence_of_sub_string(str: str, substr: str): + def remove_last_occurence_of_sub_string(string: str, substr: str) -> str: """ Delete the last occurence of substr from str String String -> String """ - index = str.rfind(substr) - return str[:index] + str[index + len(substr):] + index = string.rfind(substr) + return string[:index] + string[index + len(substr):] class PathHTMLParser(HTMLParser): currentPath = "" - paths = [] + paths = [] # type: List[str] def handle_starttag(self, tag, attrs): self.currentPath += ("/" + tag) @@ -260,7 +263,7 @@ def paths_to_text(html: str, str: str) -> List[str]: self.currentPath = remove_last_occurence_of_sub_string(self.currentPath, "/" + tag) def handle_data(self, data): - if str in data: + if string in data: self.paths.append(self.currentPath) parser = PathHTMLParser() @@ -268,7 +271,7 @@ def paths_to_text(html: str, str: str) -> List[str]: return parser.paths -def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[XSSData]: +def get_XSS_data(body: Union[str, bytes], request_URL: str, injection_point: str) -> Optional[XSSData]: """ Return a XSSDict if there is a XSS otherwise return None """ def in_script(text, index, body) -> bool: """ Whether the Numberth occurence of the first string in the second @@ -314,9 +317,9 @@ def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[ matches = regex.findall(body) for index, match in enumerate(matches): # Where the string is injected into the HTML - in_script = in_script(match, index, body) - in_HTML = in_HTML(match, index, body) - in_tag = not in_script and not in_HTML + in_script_val = in_script(match, index, body) + in_HTML_val = in_HTML(match, index, body) + in_tag = not in_script_val and not in_HTML_val in_single_quotes = inside_quote("'", match, index, body) in_double_quotes = inside_quote('"', match, index, body) # Whether you can inject: @@ -327,17 +330,17 @@ def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[ inject_slash = b"sl/bsl" in match # forward slashes inject_semi = b"se;sl" in match # semicolons inject_equals = b"eq=" in match # equals sign - if in_script and inject_slash and inject_open_angle and inject_close_angle: # e.g. <script>PAYLOAD</script> + if in_script_val and inject_slash and inject_open_angle and inject_close_angle: # e.g. <script>PAYLOAD</script> return XSSData(request_URL, injection_point, '</script><script>alert(0)</script><script>', match.decode('utf-8')) - elif in_script and in_single_quotes and inject_single_quotes and inject_semi: # e.g. <script>t='PAYLOAD';</script> + elif in_script_val and in_single_quotes and inject_single_quotes and inject_semi: # e.g. <script>t='PAYLOAD';</script> return XSSData(request_URL, injection_point, "';alert(0);g='", match.decode('utf-8')) - elif in_script and in_double_quotes and inject_double_quotes and inject_semi: # e.g. <script>t="PAYLOAD";</script> + elif in_script_val and in_double_quotes and inject_double_quotes and inject_semi: # e.g. <script>t="PAYLOAD";</script> return XSSData(request_URL, injection_point, '";alert(0);g="', @@ -380,33 +383,35 @@ def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[ injection_point, " onmouseover=alert(0) t=", match.decode('utf-8')) - elif in_HTML and not in_script and inject_open_angle and inject_close_angle and inject_slash: # e.g. <html>PAYLOAD</html> + elif in_HTML_val and not in_script_val and inject_open_angle and inject_close_angle and inject_slash: # e.g. <html>PAYLOAD</html> return XSSData(request_URL, injection_point, '<script>alert(0)</script>', match.decode('utf-8')) else: return None + return None # response is mitmproxy's entry point def response(flow: http.HTTPFlow) -> None: - cookiesDict = get_cookies(flow) + cookies_dict = get_cookies(flow) + resp = flow.response.get_text(strict=False) # Example: http://xss.guru/unclaimedScriptTag.html - find_unclaimed_URLs(flow.response.content, flow.request.url) - results = test_end_of_URL_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict) + find_unclaimed_URLs(resp, flow.request.url) + results = test_end_of_URL_injection(resp, flow.request.url, cookies_dict) log_XSS_data(results[0]) log_SQLi_data(results[1]) # Example: https://daviddworken.com/vulnerableReferer.php - results = test_referer_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict) + results = test_referer_injection(resp, flow.request.url, cookies_dict) log_XSS_data(results[0]) log_SQLi_data(results[1]) # Example: https://daviddworken.com/vulnerableUA.php - results = test_user_agent_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict) + results = test_user_agent_injection(resp, flow.request.url, cookies_dict) log_XSS_data(results[0]) log_SQLi_data(results[1]) if "?" in flow.request.url: # Example: https://daviddworken.com/vulnerable.php?name= - results = test_query_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict) + results = test_query_injection(resp, flow.request.url, cookies_dict) log_XSS_data(results[0]) log_SQLi_data(results[1]) diff --git a/examples/simple/README.md b/examples/simple/README.md index 5a7782db..d140a84c 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -1,18 +1,18 @@ ## Simple Examples -| Filename | Description | -|:-----------------------------|:---------------------------------------------------------------------------| -| add_header.py | Simple script that just adds a header to every request. | -| custom_contentview.py | Add a custom content view to the mitmproxy UI. | -| filter_flows.py | This script demonstrates how to use mitmproxy's filter pattern in scripts. | -| io_read_dumpfile.py | Read a dumpfile generated by mitmproxy. | -| io_write_dumpfile.py | Only write selected flows into a mitmproxy dumpfile. | -| log_events.py | Use mitmproxy's logging API. | -| modify_body_inject_iframe.py | Inject configurable iframe into pages. | -| modify_form.py | Modify HTTP form submissions. | -| modify_querystring.py | Modify HTTP query strings. | -| redirect_requests.py | Redirect a request to a different server. | -| script_arguments.py | Add arguments to a script. | -| send_reply_from_proxy.py | Send a HTTP response directly from the proxy. | -| upsidedownternet.py | Turn all images upside down. | -| wsgi_flask_app.py | Embed a WSGI app into mitmproxy. | +| Filename | Description | +| :----------------------------- | :--------------------------------------------------------------------------- | +| add_header.py | Simple script that just adds a header to every request. | +| custom_contentview.py | Add a custom content view to the mitmproxy UI. | +| custom_option.py | Add arguments to a script. | +| filter_flows.py | This script demonstrates how to use mitmproxy's filter pattern in scripts. | +| io_read_dumpfile.py | Read a dumpfile generated by mitmproxy. | +| io_write_dumpfile.py | Only write selected flows into a mitmproxy dumpfile. | +| log_events.py | Use mitmproxy's logging API. | +| modify_body_inject_iframe.py | Inject configurable iframe into pages. | +| modify_form.py | Modify HTTP form submissions. | +| modify_querystring.py | Modify HTTP query strings. | +| redirect_requests.py | Redirect a request to a different server. | +| send_reply_from_proxy.py | Send a HTTP response directly from the proxy. | +| upsidedownternet.py | Turn all images upside down. | +| wsgi_flask_app.py | Embed a WSGI app into mitmproxy. | diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index d684b8c7..1c8fbc05 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -88,26 +88,29 @@ class Cut: if path.startswith("+"): append = True path = mitmproxy.types.Path(path[1:]) - if len(cuts) == 1 and len(flows) == 1: - with open(path, "ab" if append else "wb") as fp: - if fp.tell() > 0: - # We're appending to a file that already exists and has content - fp.write(b"\n") - v = extract(cuts[0], flows[0]) - if isinstance(v, bytes): - fp.write(v) - else: - fp.write(v.encode("utf8")) - ctx.log.alert("Saved single cut.") - else: - with open(path, "a" if append else "w", newline='', encoding="utf8") as fp: - writer = csv.writer(fp) - for f in flows: - vals = [extract(c, f) for c in cuts] - writer.writerow( - [strutils.always_str(x) or "" for x in vals] # type: ignore - ) - ctx.log.alert("Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows))) + try: + if len(cuts) == 1 and len(flows) == 1: + with open(path, "ab" if append else "wb") as fp: + if fp.tell() > 0: + # We're appending to a file that already exists and has content + fp.write(b"\n") + v = extract(cuts[0], flows[0]) + if isinstance(v, bytes): + fp.write(v) + else: + fp.write(v.encode("utf8")) + ctx.log.alert("Saved single cut.") + else: + with open(path, "a" if append else "w", newline='', encoding="utf8") as fp: + writer = csv.writer(fp) + for f in flows: + vals = [extract(c, f) for c in cuts] + writer.writerow( + [strutils.always_str(x) or "" for x in vals] # type: ignore + ) + ctx.log.alert("Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows))) + except IOError as e: + ctx.log.error(str(e)) @command.command("cut.clip") def clip( @@ -136,4 +139,7 @@ class Cut: [strutils.always_str(v) or "" for v in vals] # type: ignore ) ctx.log.alert("Clipped %s cuts as CSV." % len(cuts)) - pyperclip.copy(fp.getvalue()) + try: + pyperclip.copy(fp.getvalue()) + except pyperclip.PyperclipException as e: + ctx.log.error(str(e)) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 0169f5b1..4bb44548 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -1,5 +1,6 @@ import typing +from mitmproxy import ctx from mitmproxy import command from mitmproxy import flow from mitmproxy import exceptions @@ -58,11 +59,14 @@ class Export(): raise exceptions.CommandError("No such export format: %s" % fmt) func = formats[fmt] # type: typing.Any v = func(f) - with open(path, "wb") as fp: - if isinstance(v, bytes): - fp.write(v) - else: - fp.write(v.encode("utf-8")) + try: + with open(path, "wb") as fp: + if isinstance(v, bytes): + fp.write(v) + else: + fp.write(v.encode("utf-8")) + except IOError as e: + ctx.log.error(str(e)) @command.command("export.clip") def clip(self, fmt: str, f: flow.Flow) -> None: @@ -73,4 +77,7 @@ class Export(): raise exceptions.CommandError("No such export format: %s" % fmt) func = formats[fmt] # type: typing.Any v = strutils.always_str(func(f)) - pyperclip.copy(v) + try: + pyperclip.copy(v) + except pyperclip.PyperclipException as e: + ctx.log.error(str(e)) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 2d030321..0a524359 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -44,13 +44,15 @@ class Script: def __init__(self, path): self.name = "scriptmanager:" + path self.path = path - self.fullpath = os.path.expanduser(path) + self.fullpath = os.path.expanduser( + path.strip("'\" ") + ) self.ns = None self.last_load = 0 self.last_mtime = 0 if not os.path.isfile(self.fullpath): - raise exceptions.OptionsError("No such script: %s" % path) + raise exceptions.OptionsError('No such script: "%s"' % self.fullpath) @property def addons(self): diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 86565b7b..9c47985c 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -253,7 +253,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): address=address, ip_address=address, cert=None, - sni=None, + sni=address[0], alpn_proto_negotiated=None, tls_version=None, source_address=('', 0), @@ -276,21 +276,21 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.wfile.write(message) self.wfile.flush() - def establish_tls(self, clientcerts, sni, **kwargs): + def establish_tls(self, *, sni=None, client_certs=None, **kwargs): if sni and not isinstance(sni, str): raise ValueError("sni must be str, not " + type(sni).__name__) - clientcert = None - if clientcerts: - if os.path.isfile(clientcerts): - clientcert = clientcerts + client_cert = None + if client_certs: + if os.path.isfile(client_certs): + client_cert = client_certs else: path = os.path.join( - clientcerts, + client_certs, self.address[0].encode("idna").decode()) + ".pem" if os.path.exists(path): - clientcert = path + client_cert = path - self.convert_to_tls(cert=clientcert, sni=sni, **kwargs) + self.convert_to_tls(cert=client_cert, sni=sni, **kwargs) self.sni = sni self.alpn_proto_negotiated = self.get_alpn_proto_negotiated() self.tls_version = self.connection.get_protocol_version_name() diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index bdab1e99..dbaa6ccc 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -49,8 +49,9 @@ def format_dict( ] entries, where key is padded to a uniform width. """ - max_key_len = max(len(k) for k in d.keys()) - max_key_len = min(max_key_len, KEY_MAX) + + max_key_len = max((len(k) for k in d.keys()), default=0) + max_key_len = min((max_key_len, KEY_MAX), default=0) for key, value in d.items(): if isinstance(key, bytes): key += b":" diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 0e43a2ac..f8eeb44b 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -13,6 +13,7 @@ import certifi from OpenSSL import SSL from kaitaistruct import KaitaiStream +import mitmproxy.options # noqa from mitmproxy import exceptions, certs from mitmproxy.contrib.kaitaistruct import tls_client_hello from mitmproxy.net import check @@ -57,6 +58,26 @@ METHOD_NAMES = { } +def client_arguments_from_options(options: "mitmproxy.options.Options") -> dict: + + if options.ssl_insecure: + verify = SSL.VERIFY_NONE + else: + verify = SSL.VERIFY_PEER + + method, tls_options = VERSION_CHOICES[options.ssl_version_server] + + return { + "verify": verify, + "method": method, + "options": tls_options, + "ca_path": options.ssl_verify_upstream_trusted_cadir, + "ca_pemfile": options.ssl_verify_upstream_trusted_ca, + "client_certs": options.client_certs, + "cipher_list": options.ciphers_server, + } + + class MasterSecretLogger: def __init__(self, filename): self.filename = filename diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 022e8133..0f3be1ea 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -9,7 +9,7 @@ from mitmproxy import http from mitmproxy import flow from mitmproxy import options from mitmproxy import connections -from mitmproxy.net import server_spec +from mitmproxy.net import server_spec, tls from mitmproxy.net.http import http1 from mitmproxy.coretypes import basethread from mitmproxy.utils import human @@ -76,8 +76,8 @@ class RequestReplayThread(basethread.BaseThread): if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_tls( - self.options.client_certs, - sni=self.f.server_conn.sni + sni=self.f.server_conn.sni, + **tls.client_arguments_from_options(self.options) ) r.first_line_format = "relative" else: @@ -91,8 +91,8 @@ class RequestReplayThread(basethread.BaseThread): server.connect() if r.scheme == "https": server.establish_tls( - self.options.client_certs, - sni=self.f.server_conn.sni + sni=self.f.server_conn.sni, + **tls.client_arguments_from_options(self.options) ) r.first_line_format = "relative" diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index d04c9801..876c1162 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -424,6 +424,9 @@ class TlsLayer(base.Layer): # * which results in garbage because the layers don' match. alpn = [self.client_conn.get_alpn_proto_negotiated()] + # We pass through the list of ciphers send by the client, because some HTTP/2 servers + # will select a non-HTTP/2 compatible cipher from our default list and then hang up + # because it's incompatible with h2. :-) ciphers_server = self.config.options.ciphers_server if not ciphers_server and self._client_tls: ciphers_server = [] @@ -432,16 +435,12 @@ class TlsLayer(base.Layer): ciphers_server.append(CIPHER_ID_NAME_MAP[id]) ciphers_server = ':'.join(ciphers_server) + args = net_tls.client_arguments_from_options(self.config.options) + args["cipher_list"] = ciphers_server self.server_conn.establish_tls( - self.config.client_certs, - self.server_sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify=self.config.openssl_verification_mode_server, - ca_path=self.config.options.ssl_verify_upstream_trusted_cadir, - ca_pemfile=self.config.options.ssl_verify_upstream_trusted_ca, - cipher_list=ciphers_server, + sni=self.server_sni, alpn_protos=alpn, + **args ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index e4a7503c..aa46501b 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -476,13 +476,16 @@ class ConsoleAddon: Save data to file as a CSV. """ rows = self._grideditor().value - with open(path, "w", newline='', encoding="utf8") as fp: - writer = csv.writer(fp) - for row in rows: - writer.writerow( - [strutils.always_str(x) or "" for x in row] # type: ignore - ) - ctx.log.alert("Saved %s rows as CSV." % (len(rows))) + try: + with open(path, "w", newline='', encoding="utf8") as fp: + writer = csv.writer(fp) + for row in rows: + writer.writerow( + [strutils.always_str(x) or "" for x in row] # type: ignore + ) + ctx.log.alert("Saved %s rows as CSV." % (len(rows))) + except IOError as e: + ctx.log.error(str(e)) @command.command("console.grideditor.editor") def grideditor_editor(self) -> None: diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index cdda3def..204820a8 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -433,7 +433,6 @@ class FocusEditor(urwid.WidgetWrap, layoutwidget.LayoutWidget): def __init__(self, master): self.master = master - self.focus_changed() def call(self, v, name, *args, **kwargs): f = getattr(v, name, None) @@ -462,7 +461,7 @@ class FocusEditor(urwid.WidgetWrap, layoutwidget.LayoutWidget): def layout_popping(self): self.call(self._w, "layout_popping") - def focus_changed(self): + def layout_pushed(self, prev): if self.master.view.focus.flow: self._w = BaseGridEditor( self.master, diff --git a/mitmproxy/tools/console/grideditor/col_subgrid.py b/mitmproxy/tools/console/grideditor/col_subgrid.py index 95995cd2..c9cbf66d 100644 --- a/mitmproxy/tools/console/grideditor/col_subgrid.py +++ b/mitmproxy/tools/console/grideditor/col_subgrid.py @@ -27,15 +27,8 @@ class Column(base.Column): ) return elif key == "m_select": - editor.master.view_grideditor( - self.subeditor( - editor.master, - editor.walker.get_current_value(), - editor.set_subeditor_value, - editor.walker.focus, - editor.walker.focus_col - ) - ) + self.subeditor.grideditor = editor + editor.master.switch_view("edit_focus_setcookie_attrs") else: return key diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index fbe48a1a..fffd782c 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,3 +1,4 @@ +import urwid import typing from mitmproxy import exceptions @@ -100,12 +101,13 @@ class CookieEditor(base.FocusEditor): flow.request.cookies = vals -class CookieAttributeEditor(base.GridEditor): +class CookieAttributeEditor(base.FocusEditor): title = "Editing Set-Cookie attributes" columns = [ col_text.Column("Name"), col_text.Column("Value"), ] + grideditor = None # type: base.BaseGridEditor def data_in(self, data): return [(k, v or "") for k, v in data] @@ -119,6 +121,20 @@ class CookieAttributeEditor(base.GridEditor): ret.append(i) return ret + def layout_pushed(self, prev): + if self.grideditor.master.view.focus.flow: + self._w = base.BaseGridEditor( + self.grideditor.master, + self.title, + self.columns, + self.grideditor.walker.get_current_value(), + self.grideditor.set_subeditor_value, + self.grideditor.walker.focus, + self.grideditor.walker.focus_col + ) + else: + self._w = urwid.Pile([]) + class SetCookieEditor(base.FocusEditor): title = "Edit SetCookie Header" diff --git a/mitmproxy/tools/console/overlay.py b/mitmproxy/tools/console/overlay.py index 55acbfdd..d255bc8c 100644 --- a/mitmproxy/tools/console/overlay.py +++ b/mitmproxy/tools/console/overlay.py @@ -40,12 +40,17 @@ class SimpleOverlay(urwid.Overlay, layoutwidget.LayoutWidget): class Choice(urwid.WidgetWrap): - def __init__(self, txt, focus, current): + def __init__(self, txt, focus, current, shortcut): + if shortcut: + selection_type = "option_selected_key" if focus else "key" + txt = [(selection_type, shortcut), ") ", txt] + else: + txt = " " + txt if current: s = "option_active_selected" if focus else "option_active" else: s = "option_selected" if focus else "text" - return super().__init__( + super().__init__( urwid.AttrWrap( urwid.Padding(urwid.Text(txt)), s, @@ -60,6 +65,8 @@ class Choice(urwid.WidgetWrap): class ChooserListWalker(urwid.ListWalker): + shortcuts = "123456789abcdefghijklmnoprstuvwxyz" + def __init__(self, choices, current): self.index = 0 self.choices = choices @@ -67,7 +74,7 @@ class ChooserListWalker(urwid.ListWalker): def _get(self, idx, focus): c = self.choices[idx] - return Choice(c, focus, c == self.current) + return Choice(c, focus, c == self.current, self.shortcuts[idx:idx + 1]) def set_focus(self, index): self.index = index @@ -87,6 +94,12 @@ class ChooserListWalker(urwid.ListWalker): return None, None return self._get(pos, False), pos + def choice_by_shortcut(self, shortcut): + for i, choice in enumerate(self.choices): + if shortcut == self.shortcuts[i:i + 1]: + return choice + return None + class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget): keyctx = "chooser" @@ -96,7 +109,8 @@ class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget): self.choices = choices self.callback = callback choicewidth = max([len(i) for i in choices]) - self.width = max(choicewidth, len(title)) + 5 + self.width = max(choicewidth, len(title)) + 7 + self.walker = ChooserListWalker(choices, current) super().__init__( urwid.AttrWrap( @@ -105,7 +119,7 @@ class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget): urwid.ListBox(self.walker), len(choices) ), - title= title + title=title ), "background" ) @@ -116,11 +130,16 @@ class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget): def keypress(self, size, key): key = self.master.keymap.handle_only("chooser", key) + choice = self.walker.choice_by_shortcut(key) + if choice: + self.callback(choice) + signals.pop_view_state.send(self) + return if key == "m_select": self.callback(self.choices[self.walker.index]) signals.pop_view_state.send(self) return - elif key == "esc": + elif key in ["q", "esc"]: signals.pop_view_state.send(self) return diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 09cfd58a..ef32b195 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -140,9 +140,10 @@ class StatusBar(urwid.WidgetWrap): signals.flowlist_change.connect(self.sig_update) master.options.changed.connect(self.sig_update) master.view.focus.sig_change.connect(self.sig_update) + master.view.sig_view_add.connect(self.sig_update) self.redraw() - def sig_update(self, sender, updated=None): + def sig_update(self, sender, flow=None, updated=None): self.redraw() def keypress(self, *args, **kwargs): diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index c7bce7d3..f2b6d3f4 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -63,6 +63,7 @@ class WindowStack: edit_focus_query = grideditor.QueryEditor(master), edit_focus_cookies = grideditor.CookieEditor(master), edit_focus_setcookies = grideditor.SetCookieEditor(master), + edit_focus_setcookie_attrs = grideditor.CookieAttributeEditor(master), edit_focus_form = grideditor.RequestFormEditor(master), edit_focus_path = grideditor.PathEditor(master), edit_focus_request_headers = grideditor.RequestHeaderEditor(master), @@ -62,25 +62,25 @@ setup( # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "blinker>=1.4, <1.5", - "brotlipy>=0.5.1, <0.8", + "brotlipy>=0.7.0,<0.8", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.2", 'h11>=0.7.0,<0.8', - "h2>=3.0, <4", - "hyperframe>=5.0, <6", - "kaitaistruct>=0.7, <0.8", + "h2>=3.0.1,<4", + "hyperframe>=5.1.0,<6", + "kaitaistruct>=0.7,<0.9", "ldap3>=2.4,<2.5", "passlib>=1.6.5, <1.8", "pyasn1>=0.3.1,<0.5", - "pyOpenSSL>=17.2,<17.6", + "pyOpenSSL>=17.5,<17.6", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.5.22, <1.7", "requests>=2.9.1, <3", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4, <1.6", "tornado>=4.3, <4.6", - "urwid>=1.3.1, <2.1", + "urwid>=2.0.1,<2.1", "wsproto>=0.11.0,<0.12.0", ], extras_require={ @@ -91,11 +91,11 @@ setup( "flake8>=3.5, <3.6", "Flask>=0.10.1, <0.13", "mypy>=0.560,<0.561", - "pytest-cov>=2.2.1, <3", - "pytest-faulthandler>=1.3.0, <2", - "pytest-timeout>=1.0.0, <2", - "pytest-xdist>=1.14, <2", - "pytest>=3.1, <4", + "pytest-cov>=2.5.1,<3", + "pytest-faulthandler>=1.3.1,<2", + "pytest-timeout>=1.2.1,<2", + "pytest-xdist>=1.22,<2", + "pytest>=3.3,<4", "rstcheck>=2.2, <4.0", "sphinx_rtd_theme>=0.1.9, <0.3", "sphinx-autobuild>=0.5.2, <0.8", diff --git a/test/examples/test_xss_scanner.py b/test/examples/test_xss_scanner.py index e15d7e10..8cf06a2a 100644 --- a/test/examples/test_xss_scanner.py +++ b/test/examples/test_xss_scanner.py @@ -343,10 +343,10 @@ class TestXSSScanner(): monkeypatch.setattr("mitmproxy.ctx.log", logger) xss.log_SQLi_data(None) assert logger.args == [] - xss.log_SQLi_data(xss.SQLiData(b'https://example.com', - b'Location', - b'Oracle.*Driver', - b'Oracle')) + xss.log_SQLi_data(xss.SQLiData('https://example.com', + 'Location', + 'Oracle.*Driver', + 'Oracle')) assert logger.args[0] == '===== SQLi Found =====' assert logger.args[1] == 'SQLi URL: https://example.com' assert logger.args[2] == 'Injection Point: Location' diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index c444b8ee..56568f21 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -7,6 +7,7 @@ from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils import pytest +import pyperclip from unittest import mock @@ -89,6 +90,13 @@ def test_cut_clip(): tctx.command(c.clip, "@all", "request.method,request.content") assert pc.called + with mock.patch('pyperclip.copy') as pc: + log_message = "Pyperclip could not find a " \ + "copy/paste mechanism for your system." + pc.side_effect = pyperclip.PyperclipException(log_message) + tctx.command(c.clip, "@all", "request.method") + assert tctx.master.has_log(log_message, level="error") + def test_cut_save(tmpdir): f = str(tmpdir.join("path")) @@ -112,6 +120,25 @@ def test_cut_save(tmpdir): assert qr(f).splitlines() == [b"GET,content", b"GET,content"] +@pytest.mark.parametrize("exception, log_message", [ + (PermissionError, "Permission denied"), + (IsADirectoryError, "Is a directory"), + (FileNotFoundError, "No such file or directory") +]) +def test_cut_save_open(exception, log_message, tmpdir): + f = str(tmpdir.join("path")) + v = view.View() + c = cut.Cut() + with taddons.context() as tctx: + tctx.master.addons.add(v, c) + v.add([tflow.tflow(resp=True)]) + + with mock.patch("mitmproxy.addons.cut.open") as m: + m.side_effect = exception(log_message) + tctx.command(c.save, "@all", "request.method", f) + assert tctx.master.has_log(log_message, level="error") + + def test_cut(): c = cut.Cut() with taddons.context(): diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index 233c62d5..07227a7a 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -1,6 +1,8 @@ -import pytest import os +import pytest +import pyperclip + from mitmproxy import exceptions from mitmproxy.addons import export # heh from mitmproxy.test import tflow @@ -94,9 +96,24 @@ def test_export(tmpdir): os.unlink(f) +@pytest.mark.parametrize("exception, log_message", [ + (PermissionError, "Permission denied"), + (IsADirectoryError, "Is a directory"), + (FileNotFoundError, "No such file or directory") +]) +def test_export_open(exception, log_message, tmpdir): + f = str(tmpdir.join("path")) + e = export.Export() + with taddons.context() as tctx: + with mock.patch("mitmproxy.addons.export.open") as m: + m.side_effect = exception(log_message) + e.file("raw", tflow.tflow(resp=True), f) + assert tctx.master.has_log(log_message, level="error") + + def test_clip(tmpdir): e = export.Export() - with taddons.context(): + with taddons.context() as tctx: with pytest.raises(exceptions.CommandError): e.clip("nonexistent", tflow.tflow(resp=True)) @@ -107,3 +124,10 @@ def test_clip(tmpdir): with mock.patch('pyperclip.copy') as pc: e.clip("curl", tflow.tflow(resp=True)) assert pc.called + + with mock.patch('pyperclip.copy') as pc: + log_message = "Pyperclip could not find a " \ + "copy/paste mechanism for your system." + pc.side_effect = pyperclip.PyperclipException(log_message) + e.clip("raw", tflow.tflow(resp=True)) + assert tctx.master.has_log(log_message, level="error") diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index c4fe6b43..78a5be6c 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -68,6 +68,18 @@ class TestScript: with pytest.raises(exceptions.OptionsError): script.Script("nonexistent") + def test_quotes_around_filename(self): + """ + Test that a script specified as '"foo.py"' works to support the calling convention of + mitmproxy 2.0, as e.g. used by Cuckoo Sandbox. + """ + path = tutils.test_data.path("mitmproxy/data/addonscripts/recorder/recorder.py") + + s = script.Script( + '"{}"'.format(path) + ) + assert '"' not in s.fullpath + def test_simple(self): with taddons.context() as tctx: sc = script.Script( diff --git a/test/mitmproxy/contentviews/test_base.py b/test/mitmproxy/contentviews/test_base.py index 777ab4dd..c94d8be2 100644 --- a/test/mitmproxy/contentviews/test_base.py +++ b/test/mitmproxy/contentviews/test_base.py @@ -1 +1,17 @@ -# TODO: write tests +import pytest +from mitmproxy.contentviews import base + + +def test_format_dict(): + d = {"one": "two", "three": "four"} + f_d = base.format_dict(d) + assert next(f_d) + + d = {"adsfa": ""} + f_d = base.format_dict(d) + assert next(f_d) + + d = {} + f_d = base.format_dict(d) + with pytest.raises(StopIteration): + next(f_d) diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 9e5d89f1..00cdbc87 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -155,7 +155,7 @@ class TestServerConnection: def test_sni(self): c = connections.ServerConnection(('', 1234)) with pytest.raises(ValueError, matches='sni must be str, not '): - c.establish_tls(None, b'foobar') + c.establish_tls(sni=b'foobar') def test_state(self): c = tflow.tserver_conn() @@ -222,17 +222,16 @@ class TestServerConnectionTLS(tservers.ServerTestBase): def handle(self): self.finish() - @pytest.mark.parametrize("clientcert", [ + @pytest.mark.parametrize("client_certs", [ None, tutils.test_data.path("mitmproxy/data/clientcert"), tutils.test_data.path("mitmproxy/data/clientcert/client.pem"), ]) - def test_tls(self, clientcert): + def test_tls(self, client_certs): c = connections.ServerConnection(("127.0.0.1", self.port)) c.connect() - c.establish_tls(clientcert, "foo.com") + c.establish_tls(client_certs=client_certs) assert c.connected() - assert c.sni == "foo.com" assert c.tls_established c.close() c.finish() diff --git a/test/mitmproxy/tools/console/test_keymap.py b/test/mitmproxy/tools/console/test_keymap.py index 00e64991..7b475ff8 100644 --- a/test/mitmproxy/tools/console/test_keymap.py +++ b/test/mitmproxy/tools/console/test_keymap.py @@ -42,7 +42,7 @@ def test_join(): km = keymap.Keymap(tctx.master) km.add("key", "str", ["options"], "help1") km.add("key", "str", ["commands"]) - return + assert len(km.bindings) == 1 assert len(km.bindings[0].contexts) == 2 assert km.bindings[0].help == "help1" @@ -27,9 +27,8 @@ commands = flake8 --jobs 8 mitmproxy pathod examples test release python test/filename_matching.py rstcheck README.rst - mypy --ignore-missing-imports ./mitmproxy - mypy --ignore-missing-imports ./pathod - mypy --ignore-missing-imports --follow-imports=skip ./examples/simple/ + mypy --ignore-missing-imports ./mitmproxy ./pathod + mypy --ignore-missing-imports --follow-imports=skip ./examples/simple/ ./examples/pathod/ ./examples/complex/ [testenv:individual_coverage] deps = diff --git a/web/package.json b/web/package.json index 31c2d6d6..77b13e8b 100644 --- a/web/package.json +++ b/web/package.json @@ -37,7 +37,8 @@ "redux-logger": "^3.0.6", "redux-mock-store": "^1.3.0", "redux-thunk": "^2.2.0", - "shallowequal": "^1.0.2" + "shallowequal": "^1.0.2", + "stable": "^0.1.6" }, "devDependencies": { "babel-core": "^6.26.0", diff --git a/web/src/js/components/FlowTable/FlowColumns.jsx b/web/src/js/components/FlowTable/FlowColumns.jsx index 02a4fba1..e60ed487 100644 --- a/web/src/js/components/FlowTable/FlowColumns.jsx +++ b/web/src/js/components/FlowTable/FlowColumns.jsx @@ -119,7 +119,7 @@ export function TimeColumn({ flow }) { return ( <td className="col-time"> {flow.response ? ( - formatTimeDelta(1000 * (flow.response.timestamp_end - flow.server_conn.timestamp_start)) + formatTimeDelta(1000 * (flow.response.timestamp_end - flow.request.timestamp_start)) ) : ( '...' )} diff --git a/web/src/js/ducks/utils/store.js b/web/src/js/ducks/utils/store.js index ac272650..ad2242ee 100644 --- a/web/src/js/ducks/utils/store.js +++ b/web/src/js/ducks/utils/store.js @@ -1,3 +1,5 @@ +import stable from 'stable' + export const SET_FILTER = 'LIST_SET_FILTER' export const SET_SORT = 'LIST_SET_SORT' export const ADD = 'LIST_ADD' @@ -35,7 +37,7 @@ export default function reduce(state = defaultState, action) { switch (action.type) { case SET_FILTER: - view = list.filter(action.filter).sort(action.sort) + view = stable(list.filter(action.filter), action.sort) viewIndex = {} view.forEach((item, index) => { viewIndex[item.id] = index @@ -43,7 +45,7 @@ export default function reduce(state = defaultState, action) { break case SET_SORT: - view = [...view].sort(action.sort) + view = stable([...view], action.sort) viewIndex = {} view.forEach((item, index) => { viewIndex[item.id] = index diff --git a/web/yarn.lock b/web/yarn.lock index aa5ae85f..1930fded 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -5449,6 +5449,10 @@ sshpk@^1.7.0: jsbn "~0.1.0"
tweetnacl "~0.14.0"
+stable@^0.1.6:
+ version "0.1.6"
+ resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.6.tgz#910f5d2aed7b520c6e777499c1f32e139fdecb10"
+
statuses@1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e"
|