diff options
104 files changed, 1233 insertions, 312 deletions
@@ -9,6 +9,7 @@ MANIFEST *.egg-info/ .coverage* .idea +.vscode .cache/ .tox*/ build/ diff --git a/.travis.yml b/.travis.yml index dca567bf..2e375078 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python branches: @@ -34,7 +33,6 @@ matrix: - python: 3.7 env: TOXENV=py37 dist: xenial - sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815 - language: node_js node_js: "node" before_install: @@ -54,7 +52,6 @@ matrix: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - pip install tox virtualenv setuptools - - pyenv global system 3.6 script: - tox after_success: @@ -67,9 +64,9 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.5 - pyenv global 3.6.5 - pyenv shell 3.6.5 + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9 + pyenv global 3.6.9 + pyenv shell 3.6.9 fi - pip install tox virtualenv setuptools @@ -106,7 +106,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test .. code-block:: bash cd test/mitmproxy/addons - pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py + pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes. diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index 4e9916ca..6a18eaf4 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -27,6 +27,6 @@ You can look at the [http][] module, or the [Request][], and [Response][] classes for other attributes that you can use when scripting. -[http][]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py +[http]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py [Request]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/request.py [Response]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/response.py diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 1dcb2f97..4e2ae47a 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -16,7 +16,7 @@ certificates have to be installed on the client device. By far the easiest way to install the mitmproxy certificates is to use the built-in certificate installation app. To do this, just start mitmproxy and configure your target device with the correct proxy settings. Now start a -browser on the device, and visit the magic domain **mitm.it**. You should see +browser on the device, and visit the magic domain [mitm.it](http://mitm.it/). You should see something like this: {{< figure src="/certinstall-webapp.png" class="has-border" >}} @@ -24,6 +24,9 @@ something like this: Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. +Note: If you are using an iOS device, you should be using the Safari browser +so that it opens the proper prompts for installing the certificate. + ## Installing the mitmproxy CA certificate manually Sometimes using the quick install app is not an option - Java or the iOS diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index fc056545..c79274bf 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -36,7 +36,7 @@ mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the major browser vendors have implemented it. Some websites are still having problems with correct HTTP/2 support in their -webservers and can cause errors, dropped connectiones, or simply no response at +webservers and can cause errors, dropped connections, or simply no response at all. We are trying to be as tolerant and forgiving as possible with the types of data we send and receive, but [some](https://github.com/mitmproxy/mitmproxy/issues/1745) diff --git a/docs/src/content/howto-ignoredomains.md b/docs/src/content/howto-ignoredomains.md index b1b14831..902a17be 100644 --- a/docs/src/content/howto-ignoredomains.md +++ b/docs/src/content/howto-ignoredomains.md @@ -72,8 +72,7 @@ method to do so: >>> mitmproxy --ignore-hosts ^example\.com:443$ {{< /highlight >}} -Here are some other examples for ignore -patterns: +Here are some other examples for ignore patterns: {{< highlight none >}} # Exempt traffic from the iOS App Store (the regex is lax, but usually just works): @@ -84,15 +83,22 @@ patterns: # Ignore example.com, but not its subdomains: --ignore-hosts '^example.com:' -# Ignore everything but example.com and mitmproxy.org: ---ignore-hosts '^(?!example\.com)(?!mitmproxy\.org)' - # Transparent mode: --ignore-hosts 17\.178\.96\.59:443 # IP address range: --ignore-hosts 17\.178\.\d+\.\d+:443 {{< / highlight >}} +This option can also be used to whitelist some domains through negative lookahead expressions. However, ignore patterns are always matched against the IP address of the target before being matched against its domain name. Thus, the pattern must allow any IP addresses using an expression like `^(?![0-9\.]+:)` in order for domains whitelisting to work. Here are examples of such patterns: + +{{< highlight none >}} +# Ignore everything but example.com and mitmproxy.org (not subdomains): +--ignore-hosts '^(?![0-9\.]+:)(?!example\.com:)(?!mitmproxy\.org:)' + +# Ignore everything but example.com and its subdomains: +--ignore-hosts '^(?![0-9\.]+:)(?!([^\.:]+\.)*example\.com:)' +{{< / highlight >}} + **Footnotes** 1. This stems from an limitation of explicit HTTP proxying: A single connection diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index ae36f579..d448bd82 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -50,7 +50,7 @@ a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com sysctl -w net.ipv4.conf.all.send_redirects=0 {{< / highlight >}} -If your test device is on the same physical network, your machine shouldn't inform the device that +If your test device is on the same physical network, your machine shouldn't inform the device that there's a shorter route available by skipping the proxy. If you want to persist this across reboots, see above. @@ -83,9 +83,34 @@ The `--mode transparent` option turns on transparent mode, and the `--showhost` ### 5. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). +### Work-around to redirect traffic originating from the machine itself + +Follow steps **1, 2** as above, but *instead* of the commands in step **3**, run the following + +Create a user to run the mitmproxy + +{{< highlight bash >}} +sudo useradd --create-home mitmproxyuser +sudo -u mitmproxyuser bash -c 'cd ~ && pip install --user mitmproxy' +{{< / highlight >}} + +Then, configure the iptables rules to redirect all traffic from our local machine to mitmproxy. **Note**, as soon as you run these, you won't be able to perform successful network calls *until* you start mitmproxy. If you run into issues, `iptables -t nat -F` is a heavy handed way to flush (clear) *all* the rules from the iptables `nat` table (which includes any other rules you had configured). + +{{< highlight bash >}} +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +{{< / highlight >}} + +This will redirect the packets from all users other than `mitmproxyuser` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `mitmproxyuser`. Hence step **4** should look like: +{{< highlight bash >}} +sudo -u mitmproxyuser bash -c '$HOME/.local/bin/mitmproxy --mode transparent --showhost --set block_global=false' +{{< / highlight >}} + ## OpenBSD @@ -124,7 +149,7 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy --mode transparent --showhost +mitmproxy --mode transparent --listen-host 127.0.0.1 --showhost {{< / highlight >}} The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells @@ -132,7 +157,7 @@ mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). @@ -213,7 +238,7 @@ mitmproxy to use the value of the Host header for URL display. ### 7. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). {{% note %}} @@ -229,7 +254,7 @@ for more. ### Work-around to redirect traffic originating from the machine itself -Follow the steps **1, 2** as above. In step **3** change the contents of the file **pf.conf** to +Follow steps **1, 2** as above, but in step **2** change the contents of the file **pf.conf** to {{< highlight none >}} #The ports to redirect to proxy @@ -246,18 +271,12 @@ tproxy_user = "nobody" #This cannot involve the user which runs the #transparent proxy as that would cause an infinite loop. # -#Here we redirect for all users which don't run transparent proxy. -redir_users = "{ !=" $tproxy_user "}" - -#If you only wish to redirect traffic for particular users -#you may also do: -#redir_users = "{= john, = jane}" rdr pass proto tcp from any to any port $redir_ports -> $tproxy -pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user $redir_users +pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user { != $tproxy_user } {{< / highlight >}} -Follow steps **4-6** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **7** should look like: +Follow steps **3-5** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **6** should look like: {{< highlight bash >}} sudo -u nobody mitmproxy --mode transparent --showhost diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md index 588223ac..a55d177b 100644 --- a/docs/src/content/howto-wireshark-tls.md +++ b/docs/src/content/howto-wireshark-tls.md @@ -7,7 +7,7 @@ menu: # Wireshark and SSL/TLS Master Secrets -The SSL/SSL master keys can be logged by mitmproxy so that external programs can +The SSL/TLS master keys can be logged by mitmproxy so that external programs can decrypt SSL/TLS connections both from and to the proxy. Recent versions of Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. diff --git a/docs/src/content/tute-highscores.md b/docs/src/content/tute-highscores.md index f5cbd7bc..2d03076d 100644 --- a/docs/src/content/tute-highscores.md +++ b/docs/src/content/tute-highscores.md @@ -67,7 +67,7 @@ timestamp. Looks pretty simple to mess with. Lets edit the score submission. First, select it in mitmproxy, then press <span data-role="kbd">enter</span> to view it. Make sure you're -viewing the request, not the response -you can use +viewing the request, not the response - you can use <span data-role="kbd">tab</span> to flick between the two. Now press <span data-role="kbd">e</span> for edit. You'll be prompted for the part of the request you want to change - press <span data-role="kbd">r</span> diff --git a/examples/addons/commands-paths.py b/examples/addons/commands-paths.py index f37a0fbc..4d9535b9 100644 --- a/examples/addons/commands-paths.py +++ b/examples/addons/commands-paths.py @@ -20,9 +20,9 @@ class MyAddon: for f in flows: totals[f.request.host] = totals.setdefault(f.request.host, 0) + 1 - fp = open(path, "w+") - for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): - fp.write("%s: %s\n" % (cnt, dom)) + with open(path, "w+") as fp: + for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): + fp.write("%s: %s\n" % (cnt, dom)) ctx.log.alert("done") diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index a3c1a017..63222eae 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -15,7 +15,7 @@ Usage: # Used as the target location if neither SNI nor host header are present. --mode reverse:http://example.com/ # To avoid auto rewriting of host header by the reverse proxy target. - --set keep-host-header + --set keep_host_header mitmdump -p 80 --mode reverse:http://localhost:443/ diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 33a2f79f..e3cea9fd 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,10 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + timings = { + k: int(1000 * v) if v != -1 else -1 + for k, v in timings_raw.items() + } # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index c862536f..69b9ea9e 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -38,7 +38,7 @@ def response(flow: http.HTTPFlow) -> None: flow.response.content = flow.response.content.replace(b'https://', b'http://') # strip meta tag upgrade-insecure-requests in response body - csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' + csp_meta_tag_pattern = br'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) # strip links in 'Location' header @@ -52,7 +52,7 @@ def response(flow: http.HTTPFlow) -> None: # strip upgrade-insecure-requests in Content-Security-Policy header if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) # strip secure flag from 'Set-Cookie' headers cookies = flow.response.headers.get_all('Set-Cookie') diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index cdaaf478..97e94ed4 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -1,4 +1,4 @@ -""" +r""" __ __ _____ _____ _____ \ \ / // ____/ ____| / ____| diff --git a/examples/simple/README.md b/examples/simple/README.md index 2fafdd5a..66a05b30 100644 --- a/examples/simple/README.md +++ b/examples/simple/README.md @@ -8,6 +8,7 @@ | filter_flows.py | This script demonstrates how to use mitmproxy's filter pattern in scripts. | | io_read_dumpfile.py | Read a dumpfile generated by mitmproxy. | | io_write_dumpfile.py | Only write selected flows into a mitmproxy dumpfile. | +| link_expander.py | Discover relative links in HTML traffic and replace them with absolute paths | | log_events.py | Use mitmproxy's logging API. | | modify_body_inject_iframe.py | Inject configurable iframe into pages. | | modify_form.py | Modify HTTP form submissions. | diff --git a/examples/simple/link_expander.py b/examples/simple/link_expander.py new file mode 100644 index 00000000..0edf7c98 --- /dev/null +++ b/examples/simple/link_expander.py @@ -0,0 +1,28 @@ +# This script determines if request is an HTML webpage and if so seeks out +# relative links (<a href="./about.html">) and expands them to absolute links +# In practice this can be used to front an indexing spider that may not have the capability to expand relative page links. +# Usage: mitmdump -s link_expander.py or mitmproxy -s link_expander.py + +import re +from urllib.parse import urljoin + + +def response(flow): + + if "Content-Type" in flow.response.headers and flow.response.headers["Content-Type"].find("text/html") != -1: + pageUrl = flow.request.url + pageText = flow.response.text + pattern = (r"<a\s+(?:[^>]*?\s+)?href=(?P<delimiter>[\"'])" + r"(?P<link>(?!https?:\/\/|ftps?:\/\/|\/\/|#|javascript:|mailto:).*?)(?P=delimiter)") + rel_matcher = re.compile(pattern, flags=re.IGNORECASE) + rel_matches = rel_matcher.finditer(pageText) + map_dict = {} + for match_num, match in enumerate(rel_matches): + (delimiter, rel_link) = match.group("delimiter", "link") + abs_link = urljoin(pageUrl, rel_link) + map_dict["{0}{1}{0}".format(delimiter, rel_link)] = "{0}{1}{0}".format(delimiter, abs_link) + for map in map_dict.items(): + pageText = pageText.replace(*map) + # Uncomment the following to print the expansion mapping + # print("{0} -> {1}".format(*map)) + flow.response.text = pageText
\ No newline at end of file diff --git a/examples/simple/websocket_messages.py b/examples/simple/websocket_messages.py new file mode 100644 index 00000000..719e7b10 --- /dev/null +++ b/examples/simple/websocket_messages.py @@ -0,0 +1,13 @@ +import re +from mitmproxy import ctx + + +def websocket_message(flow): + # get the latest message + message = flow.messages[-1] + + # simply print the content of the message + ctx.log.info(message.content) + + # manipulate the message content + message.content = re.sub(r'^Hello', 'HAPPY', message.content) diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 645f3a93..8a565a73 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -42,7 +42,7 @@ def safecall(): yield except (exceptions.AddonHalt, exceptions.OptionsError): raise - except Exception as e: + except Exception: etype, value, tb = sys.exc_info() tb = cut_traceback(tb, "invoke_addon") ctx.log.error( @@ -184,7 +184,7 @@ class AddonManager: raise exceptions.AddonManagerError("No such addon: %s" % n) self.chain = [i for i in self.chain if i is not a] del self.lookup[_get_name(a)] - self.invoke_addon(a, "done") + self.invoke_addon(addon, "done") def __len__(self): return len(self.chain) diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py index 91f9f709..4ccde0e1 100644 --- a/mitmproxy/addons/block.py +++ b/mitmproxy/addons/block.py @@ -36,4 +36,4 @@ class Block: layer.reply.kill() if ctx.options.block_global and address.is_global: ctx.log.warn("Client connection from %s killed by block_global" % astr) - layer.reply.kill()
\ No newline at end of file + layer.reply.kill() diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 99ba1d8d..c56c0e74 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,6 +1,7 @@ import queue import threading import typing +import time from mitmproxy import log from mitmproxy import controller @@ -94,6 +95,7 @@ class RequestReplayThread(basethread.BaseThread): server.wfile.write(http1.assemble_request(r)) server.wfile.flush() + r.timestamp_start = r.timestamp_end = time.time() if f.server_conn: f.server_conn.close() diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 0322b4ce..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -53,7 +53,7 @@ class Core: if "body_size_limit" in updated: try: human.parse_size(opts.body_size_limit) - except ValueError as e: + except ValueError: raise exceptions.OptionsError( "Invalid body size limit specification: %s" % opts.body_size_limit @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 87f45976..dcac6b82 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -27,9 +27,10 @@ def colorful(line, styles): class Dumper: - def __init__(self, outfile=sys.stdout): + def __init__(self, outfile=sys.stdout, errfile=sys.stderr): self.filter: flowfilter.TFilter = None self.outfp: typing.io.TextIO = outfile + self.errfp: typing.io.TextIO = errfile def load(self, loader): loader.add_option( @@ -70,6 +71,11 @@ class Dumper: if self.outfp: self.outfp.flush() + def echo_error(self, text, **style): + click.secho(text, file=self.errfp, **style) + if self.errfp: + self.errfp.flush() + def _echo_headers(self, headers): for k, v in headers.fields: k = strutils.bytes_to_escaped_str(k) @@ -243,7 +249,7 @@ class Dumper: self.echo_flow(f) def websocket_error(self, f): - self.echo( + self.echo_error( "Error in WebSocket connection to {}: {}".format( human.format_address(f.server_conn.address), f.error ), @@ -268,7 +274,7 @@ class Dumper: f.close_reason)) def tcp_error(self, f): - self.echo( + self.echo_error( "Error in TCP connection to {}: {}".format( human.format_address(f.server_conn.address), f.error ), diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index f8fc344a..abdc1eac 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -152,7 +152,7 @@ class ProxyAuth: p = ctx.options.proxyauth[1:] try: self.htpasswd = passlib.apache.HtpasswdFile(p) - except (ValueError, OSError) as v: + except (ValueError, OSError): raise exceptions.OptionsError( "Could not open htpasswd file: %s" % p ) diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py index 63e382ec..f9073c3e 100644 --- a/mitmproxy/addons/session.py +++ b/mitmproxy/addons/session.py @@ -87,8 +87,8 @@ class SessionDB: def _create_session(self): script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() - self.con.executescript(qry) + with open(script_path, 'r') as qry: + self.con.executescript(qry.read()) self.con.commit() @staticmethod diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 1c8bd0ce..8d27840f 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -103,7 +103,7 @@ orders = [ ] -class View(collections.Sequence): +class View(collections.abc.Sequence): def __init__(self): super().__init__() self._store = collections.OrderedDict() @@ -638,7 +638,7 @@ class Focus: self.flow = flow -class Settings(collections.Mapping): +class Settings(collections.abc.Mapping): def __init__(self, view: View) -> None: self.view = view self._values: typing.MutableMapping[str, typing.Dict] = {} diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 58aea6d5..6f5f8c09 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -5,6 +5,7 @@ import datetime import ipaddress import sys import typing +import contextlib from pyasn1.type import univ, constraint, char, namedtype, tag from pyasn1.codec.der.decoder import decode @@ -35,14 +36,14 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= """ -def create_ca(o, cn, exp): +def create_ca(organization, cn, exp): key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) cert = OpenSSL.crypto.X509() cert.set_serial_number(int(time.time() * 10000)) cert.set_version(2) cert.get_subject().CN = cn - cert.get_subject().O = o + cert.get_subject().O = organization cert.gmtime_adj_notBefore(-3600 * 48) cert.gmtime_adj_notAfter(exp) cert.set_issuer(cert.get_subject()) @@ -79,7 +80,7 @@ def create_ca(o, cn, exp): return key, cert -def dummy_cert(privkey, cacert, commonname, sans): +def dummy_cert(privkey, cacert, commonname, sans, organization): """ Generates a dummy certificate. @@ -87,6 +88,7 @@ def dummy_cert(privkey, cacert, commonname, sans): cacert: CA certificate commonname: Common name for the generated certificate. sans: A list of Subject Alternate Names. + organization: Organization name for the generated certificate. Returns cert if operation succeeded, None if not. """ @@ -106,6 +108,8 @@ def dummy_cert(privkey, cacert, commonname, sans): cert.set_issuer(cacert.get_subject()) if commonname is not None and len(commonname) < 64: cert.get_subject().CN = commonname + if organization is not None: + cert.get_subject().O = organization cert.set_serial_number(int(time.time() * 10000)) if ss: cert.set_version(2) @@ -196,16 +200,31 @@ class CertStore: return cls(key, ca, ca_path, dh) @staticmethod - def create_store(path, basename, o=None, cn=None, expiry=DEFAULT_EXP): + @contextlib.contextmanager + def umask_secret(): + """ + Context to temporarily set umask to its original value bitor 0o77. + Useful when writing private keys to disk so that only the owner + will be able to read them. + """ + original_umask = os.umask(0) + os.umask(original_umask | 0o77) + try: + yield + finally: + os.umask(original_umask) + + @staticmethod + def create_store(path, basename, organization=None, cn=None, expiry=DEFAULT_EXP): if not os.path.exists(path): os.makedirs(path) - o = o or basename + organization = organization or basename cn = cn or basename - key, ca = create_ca(o=o, cn=cn, exp=expiry) + key, ca = create_ca(organization=organization, cn=cn, exp=expiry) # Dump the CA plus private key - with open(os.path.join(path, basename + "-ca.pem"), "wb") as f: + with CertStore.umask_secret(), open(os.path.join(path, basename + "-ca.pem"), "wb") as f: f.write( OpenSSL.crypto.dump_privatekey( OpenSSL.crypto.FILETYPE_PEM, @@ -236,7 +255,7 @@ class CertStore: f.write(p12.export()) # Dump the certificate and key in a PKCS12 format for Windows devices - with open(os.path.join(path, basename + "-ca.p12"), "wb") as f: + with CertStore.umask_secret(), open(os.path.join(path, basename + "-ca.p12"), "wb") as f: p12 = OpenSSL.crypto.PKCS12() p12.set_certificate(ca) p12.set_privatekey(key) @@ -289,7 +308,7 @@ class CertStore: ret.append(b"*." + b".".join(parts[i:])) return ret - def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes]): + def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], organization: typing.Optional[bytes] = None): """ Returns an (cert, privkey, cert_chain) tuple. @@ -297,6 +316,8 @@ class CertStore: valid, plain-ASCII, IDNA-encoded domain name. sans: A list of Subject Alternate Names. + + organization: Organization name for the generated certificate. """ potential_keys: typing.List[TCertId] = [] @@ -319,7 +340,8 @@ class CertStore: self.default_privatekey, self.default_ca, commonname, - sans), + sans, + organization), privatekey=self.default_privatekey, chain_file=self.default_chain_file) self.certs[(commonname, tuple(sans))] = entry @@ -432,6 +454,14 @@ class Cert(serializable.Serializable): return c @property + def organization(self): + c = None + for i in self.subject: + if i[0] == b"O": + c = i[1] + return c + + @property def altnames(self): """ Returns: diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 7810535e..01c6d221 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -24,7 +24,7 @@ from . import ( auto, raw, hex, json, xml_html, wbxml, javascript, css, urlencoded, multipart, image, query, protobuf ) -from .base import View, VIEW_CUTOFF, KEY_MAX, format_text, format_dict, TViewResult +from .base import View, KEY_MAX, format_text, format_dict, TViewResult views: List[View] = [] content_types_map: Dict[str, List[View]] = {} @@ -160,6 +160,6 @@ add(query.ViewQuery()) add(protobuf.ViewProtobuf()) __all__ = [ - "View", "VIEW_CUTOFF", "KEY_MAX", "format_text", "format_dict", "TViewResult", + "View", "KEY_MAX", "format_text", "format_dict", "TViewResult", "get", "add", "remove", "get_content_view", "get_message_content_view", ] diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 260c7af8..9b34f3d4 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -1,8 +1,6 @@ # Default view cutoff *in lines* import typing -VIEW_CUTOFF = 512 - KEY_MAX = 30 TTextType = typing.Union[str, bytes] # FIXME: This should be either bytes or str ultimately. @@ -37,32 +35,52 @@ class View: raise NotImplementedError() # pragma: no cover -def format_dict( - d: typing.Mapping[TTextType, TTextType] +def format_pairs( + items: typing.Iterable[typing.Tuple[TTextType, TTextType]] ) -> typing.Iterator[TViewLine]: + """ - Helper function that transforms the given dictionary into a list of + Helper function that accepts a list of (k,v) pairs into a list of [ - ("key", key ) + ("key", key ) ("value", value) ] - entries, where key is padded to a uniform width. + where key is padded to a uniform width """ - max_key_len = max((len(k) for k in d.keys()), default=0) + max_key_len = max((len(k[0]) for k in items), default=0) max_key_len = min((max_key_len, KEY_MAX), default=0) - for key, value in d.items(): + + for key, value in items: if isinstance(key, bytes): + key += b":" else: key += ":" + key = key.ljust(max_key_len + 2) + yield [ ("header", key), ("text", value) ] +def format_dict( + d: typing.Mapping[TTextType, TTextType] +) -> typing.Iterator[TViewLine]: + """ + Helper function that transforms the given dictionary into a list of + [ + ("key", key ) + ("value", value) + ] + entries, where key is padded to a uniform width. + """ + + return format_pairs(d.items()) + + def format_text(text: TTextType) -> typing.Iterator[TViewLine]: """ Helper function that transforms bytes into the view output format. diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index cbe8ce62..44b33761 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -16,7 +16,7 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are: CSS_SPECIAL_AREAS = ( "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", "//" + strutils.SINGLELINE_CONTENT + "$" ) CSS_SPECIAL_CHARS = "{};:" diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index fcc50cb5..d5bb404f 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata: entries = block.body.body.entries for entry in entries: comment = entry.bytes - if comment is not b'': + if comment != b'': parts.append(('comment', str(comment))) return parts diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index 1440ea5d..b5f09150 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -10,9 +10,9 @@ SPECIAL_AREAS = ( r"'" + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + "'", r'"' + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + '"', r'`' + strutils.MULTILINE_CONTENT + strutils.NO_ESCAPE + '`', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", r"//" + strutils.SINGLELINE_CONTENT + "$", - r"for\(" + strutils.SINGLELINE_CONTENT + "\)", + r"for\(" + strutils.SINGLELINE_CONTENT + r"\)", ) diff --git a/mitmproxy/contentviews/query.py b/mitmproxy/contentviews/query.py index 4896624b..0f74ea2f 100644 --- a/mitmproxy/contentviews/query.py +++ b/mitmproxy/contentviews/query.py @@ -9,6 +9,6 @@ class ViewQuery(base.View): def __call__(self, data, **metadata): query = metadata.get("query") if query: - return "Query", base.format_dict(query) + return "Query", base.format_pairs(query.items(multi=True)) else: return "Query", base.format_text("") diff --git a/mitmproxy/contentviews/urlencoded.py b/mitmproxy/contentviews/urlencoded.py index 371a160e..e35bbdb7 100644 --- a/mitmproxy/contentviews/urlencoded.py +++ b/mitmproxy/contentviews/urlencoded.py @@ -1,5 +1,4 @@ from mitmproxy.net.http import url -from mitmproxy.coretypes import multidict from . import base @@ -13,4 +12,4 @@ class ViewURLEncoded(base.View): except ValueError: return None d = url.decode(data) - return "URLEncoded form", base.format_dict(multidict.MultiDict(d)) + return "URLEncoded form", base.format_pairs(d) diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 658fbcd7..00a62a15 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -18,7 +18,7 @@ The implementation is split into two main parts: tokenization and formatting of """ # http://www.xml.com/pub/a/2001/07/25/namingparts.html - this is close enough for what we do. -REGEX_TAG = re.compile("[a-zA-Z0-9._:\-]+(?!=)") +REGEX_TAG = re.compile(r"[a-zA-Z0-9._:\-]+(?!=)") # https://www.w3.org/TR/html5/syntax.html#void-elements HTML_VOID_ELEMENTS = { "area", "base", "br", "col", "embed", "hr", "img", "input", "keygen", "link", "meta", "param", diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 8a6e7a2b..88ce4e54 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2be() self.ifd0_ofs = self._io.read_u4be() @@ -25,6 +24,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2be() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifBe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2be()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2be()) self.length = self._io.read_u4be() @@ -552,7 +560,7 @@ class ExifBe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 84e53a38..e25a2fc9 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2le() self.ifd0_ofs = self._io.read_u4le() @@ -25,6 +24,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2le() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifLe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2le()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2le()) self.length = self._io.read_u4le() @@ -552,7 +560,7 @@ class ExifLe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/wbxml/ASCommandResponse.py b/mitmproxy/contrib/wbxml/ASCommandResponse.py index 2d60eb2d..34755cbe 100644 --- a/mitmproxy/contrib/wbxml/ASCommandResponse.py +++ b/mitmproxy/contrib/wbxml/ASCommandResponse.py @@ -63,8 +63,9 @@ if __name__ == "__main__": listOfSamples = os.listdir(samplesDir) for filename in listOfSamples: - byteWBXML = open(samplesDir + os.sep + filename, "rb").read() - + with open(samplesDir + os.sep + filename, "rb") as f: + byteWBXML = f.read() + logging.info("-"*100) logging.info(filename) logging.info("-"*100) diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 7f8df96f..0d8f1062 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -475,7 +475,30 @@ def _make(): parts.append(f) simplerex = "".join(c for c in pp.printables if c not in "()~'\"") + alphdevanagari = pp.pyparsing_unicode.Devanagari.alphas + alphcyrillic = pp.pyparsing_unicode.Cyrillic.alphas + alphgreek = pp.pyparsing_unicode.Greek.alphas + alphchinese = pp.pyparsing_unicode.Chinese.alphas + alpharabic = pp.pyparsing_unicode.Arabic.alphas + alphhebrew = pp.pyparsing_unicode.Hebrew.alphas + alphjapanese = pp.pyparsing_unicode.Japanese.alphas + alphkorean = pp.pyparsing_unicode.Korean.alphas + alphlatin1 = pp.pyparsing_unicode.Latin1.alphas + alphlatinA = pp.pyparsing_unicode.LatinA.alphas + alphlatinB = pp.pyparsing_unicode.LatinB.alphas + rex = pp.Word(simplerex) |\ + pp.Word(alphcyrillic) |\ + pp.Word(alphgreek) |\ + pp.Word(alphchinese) |\ + pp.Word(alpharabic) |\ + pp.Word(alphdevanagari) |\ + pp.Word(alphhebrew) |\ + pp.Word(alphjapanese) |\ + pp.Word(alphkorean) |\ + pp.Word(alphlatin1) |\ + pp.Word(alphlatinA) |\ + pp.Word(alphlatinB) |\ pp.QuotedString("\"", escChar='\\') |\ pp.QuotedString("'", escChar='\\') for klass in filter_rex: diff --git a/mitmproxy/master.py b/mitmproxy/master.py index ecb8090b..e5683356 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -84,13 +84,14 @@ class Master: exc = None try: loop() - except Exception as e: # pragma: no cover + except Exception: # pragma: no cover exc = traceback.format_exc() finally: if not self.should_exit.is_set(): # pragma: no cover self.shutdown() loop = asyncio.get_event_loop() - for p in asyncio.Task.all_tasks(): + tasks = asyncio.all_tasks(loop) if sys.version_info >= (3, 7) else asyncio.Task.all_tasks(loop) + for p in tasks: p.cancel() loop.close() diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index aaea851f..a19ad6fe 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -2,7 +2,7 @@ import ipaddress import re # Allow underscore in host name -_label_valid = re.compile(b"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) +_label_valid = re.compile(br"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) def is_valid_host(host: bytes) -> bool: diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 1472ab55..2745701f 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -304,7 +304,7 @@ def refresh_set_cookie_header(c: str, delta: int) -> str: e = email.utils.parsedate_tz(attrs["expires"]) if e: f = email.utils.mktime_tz(e) + delta - attrs.set_all("expires", [email.utils.formatdate(f)]) + attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)]) else: # This can happen when the expires tag is invalid. # reddit.com sends a an expires tag like this: "Thu, 31 Dec diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 8cb96e5c..16d399ca 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -9,6 +9,7 @@ from io import BytesIO import gzip import zlib import brotli +import zstandard as zstd from typing import Union, Optional, AnyStr # noqa @@ -52,7 +53,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes: return brotli.compress(content) +def decode_zstd(content: bytes) -> bytes: + if not content: + return b"" + zstd_ctx = zstd.ZstdDecompressor() + try: + return zstd_ctx.decompress(content) + except zstd.ZstdError: + # If the zstd stream is streamed without a size header, + # try decoding with a 10MiB output buffer + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) + + +def encode_zstd(content: bytes) -> bytes: + zstd_ctx = zstd.ZstdCompressor() + return zstd_ctx.compress(content) + + def decode_deflate(content: bytes) -> bytes: """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -170,6 +188,7 @@ custom_decode = { "gzip": decode_gzip, "deflate": decode_deflate, "br": decode_brotli, + "zstd": decode_zstd, } custom_encode = { "none": identity, @@ -177,6 +196,7 @@ custom_encode = { "gzip": encode_gzip, "deflate": encode_deflate, "br": encode_brotli, + "zstd": encode_zstd, } __all__ = ["encode", "decode"] diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 06d00377..6830c6cd 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -68,7 +68,7 @@ class Message(serializable.Serializable): @property def raw_content(self) -> bytes: """ - The raw (encoded) HTTP message body + The raw (potentially compressed) HTTP message body as bytes. See also: :py:attr:`content`, :py:class:`text` """ @@ -80,10 +80,10 @@ class Message(serializable.Serializable): def get_content(self, strict: bool=True) -> bytes: """ - The HTTP message body decoded with the content-encoding header (e.g. gzip) + The uncompressed HTTP message body as bytes. Raises: - ValueError, when the content-encoding is invalid and strict is True. + ValueError, when the HTTP content-encoding is invalid and strict is True. See also: :py:class:`raw_content`, :py:attr:`text` """ @@ -165,22 +165,26 @@ class Message(serializable.Serializable): return ct[2].get("charset") return None - def _guess_encoding(self) -> str: + def _guess_encoding(self, content=b"") -> str: enc = self._get_content_type_charset() - if enc: - return enc - - if "json" in self.headers.get("content-type", ""): - return "utf8" - else: - # We may also want to check for HTML meta tags here at some point. - # REGEX_ENCODING = re.compile(rb"""<meta[^>]+charset=['"]?([^'"]+)""") - return "latin-1" + if not enc: + if "json" in self.headers.get("content-type", ""): + enc = "utf8" + if not enc: + meta_charset = re.search(rb"""<meta[^>]+charset=['"]?([^'">]+)""", content) + if meta_charset: + enc = meta_charset.group(1).decode("ascii", "ignore") + if not enc: + enc = "latin-1" + # Use GB 18030 as the superset of GB2312 and GBK to fix common encoding problems on Chinese websites. + if enc.lower() in ("gb2312", "gbk"): + enc = "gb18030" + + return enc def get_text(self, strict: bool=True) -> Optional[str]: """ - The HTTP message body decoded with both content-encoding header (e.g. gzip) - and content-type header charset. + The uncompressed and decoded HTTP message body as text. Raises: ValueError, when either content-encoding or charset is invalid and strict is True. @@ -189,9 +193,9 @@ class Message(serializable.Serializable): """ if self.raw_content is None: return None - enc = self._guess_encoding() content = self.get_content(strict) + enc = self._guess_encoding(content) try: return encoding.decode(content, enc) except ValueError: @@ -232,7 +236,7 @@ class Message(serializable.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..ef33ca49 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +import time from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union from mitmproxy.coretypes import multidict @@ -101,6 +102,7 @@ class Request(message.Message): ) req.url = url + req.timestamp_start = time.time() # Headers can be list or dict, we differentiate here. if isinstance(headers, dict): @@ -421,7 +423,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate", "br"} + for e in {"gzip", "identity", "deflate", "br", "zstd"} if e in accept_encoding ) ) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 48527d63..9491fc03 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -186,7 +186,7 @@ class Response(message.Message): d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = formatdate(new) + self.headers[i] = formatdate(new, usegmt=True) c = [] for set_cookie_header in self.headers.get_all("set-cookie"): try: diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index f938cb12..d8e14aeb 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -21,16 +21,25 @@ def parse(url): Raises: ValueError, if the URL is not properly formatted. """ - parsed = urllib.parse.urlparse(url) + # Size of Ascii character after encoding is 1 byte which is same as its size + # But non-Ascii character's size after encoding will be more than its size + def ascii_check(l): + if len(l) == len(str(l).encode()): + return True + return False + + if isinstance(url, bytes): + url = url.decode() + if not ascii_check(url): + url = urllib.parse.urlsplit(url) + url = list(url) + url[3] = urllib.parse.quote(url[3]) + url = urllib.parse.urlunsplit(url) + parsed = urllib.parse.urlparse(url) if not parsed.hostname: raise ValueError("No hostname given") - if isinstance(url, bytes): - host = parsed.hostname - - # this should not raise a ValueError, - # but we try to be very forgiving here and accept just everything. else: host = parsed.hostname.encode("idna") if isinstance(parsed, urllib.parse.ParseResult): diff --git a/mitmproxy/net/websockets/masker.py b/mitmproxy/net/websockets/masker.py index 47b1a688..6134e09e 100644 --- a/mitmproxy/net/websockets/masker.py +++ b/mitmproxy/net/websockets/masker.py @@ -1,3 +1,6 @@ +import sys + + class Masker: """ Data sent from the server must be masked to prevent malicious clients @@ -12,12 +15,13 @@ class Masker: self.offset = 0 def mask(self, offset, data): - result = bytearray(data) - for i in range(len(data)): - result[i] ^= self.key[offset % 4] - offset += 1 - result = bytes(result) - return result + datalen = len(data) + offset_mod = offset % 4 + data = int.from_bytes(data, sys.byteorder) + num_keys = (datalen + offset_mod + 3) // 4 + mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen + + offset_mod], sys.byteorder) + return (data ^ mask).to_bytes(datalen, sys.byteorder) def __call__(self, data): ret = self.mask(self.offset, data) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 047a44cd..56146153 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -6,6 +6,7 @@ from mitmproxy.net import tls CONF_DIR = "~/.mitmproxy" LISTEN_PORT = 8080 +CONTENT_VIEW_LINES_CUTOFF = 512 class Options(optmanager.OptManager): @@ -67,6 +68,10 @@ class Options(optmanager.OptManager): """ ) self.add_option( + "allow_hosts", Sequence[str], [], + "Opposite of --ignore-hosts." + ) + self.add_option( "listen_host", str, "", "Address to bind proxy to." ) @@ -161,5 +166,12 @@ class Options(optmanager.OptManager): communication contents are printed to the log in verbose mode. """ ) + self.add_option( + "content_view_lines_cutoff", int, CONTENT_VIEW_LINES_CUTOFF, + """ + Flow content view lines limit. Limit is enabled by default to + speedup flows browsing. + """ + ) self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 06e696c0..6e187b0d 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -320,7 +320,9 @@ class OptManager: update = {} for optname, optval in self.deferred.items(): if optname in self._options: - update[optname] = self.parse_setval(self._options[optname], optval) + if isinstance(optval, str): + optval = self.parse_setval(self._options[optname], optval) + update[optname] = optval self.update(**update) for k in update.keys(): del self.deferred[k] diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index bb5eb515..5e22ec31 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -11,7 +11,7 @@ def lookup(address, port, s): """ # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() spec = "%s:%s" % (address, port) for i in s.split("\n"): diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index b849afa5..19d9abd4 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -13,6 +13,7 @@ import typing import click import collections +import collections.abc import pydivert import pydivert.consts @@ -58,7 +59,7 @@ class Resolver: def original_addr(self, csock: socket.socket): ip, port = csock.getpeername()[:2] - ip = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) + ip = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) ip = ip.split("%", 1)[0] with self.lock: try: @@ -171,7 +172,7 @@ def MIB_TCPTABLE_OWNER_PID(size): TCP_TABLE_OWNER_PID_CONNECTIONS = 4 -class TcpConnectionTable(collections.Mapping): +class TcpConnectionTable(collections.abc.Mapping): DEFAULT_TABLE_SIZE = 4096 def __init__(self): diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index f32d3086..75e372ae 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -14,7 +14,8 @@ CONF_BASENAME = "mitmproxy" class HostMatcher: - def __init__(self, patterns=tuple()): + def __init__(self, handle, patterns=tuple()): + self.handle = handle self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -22,10 +23,10 @@ class HostMatcher: if not address: return False host = "%s:%s" % address - if any(rex.search(host) for rex in self.regexes): - return True - else: - return False + if self.handle in ["ignore", "tcp"]: + return any(rex.search(host) for rex in self.regexes) + else: # self.handle == "allow" + return any(not rex.search(host) for rex in self.regexes) def __bool__(self): return bool(self.patterns) @@ -36,7 +37,7 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore: HostMatcher = None + self.check_filter: HostMatcher = None self.check_tcp: HostMatcher = None self.certstore: certs.CertStore = None self.upstream_server: typing.Optional[server_spec.ServerSpec] = None @@ -44,10 +45,18 @@ class ProxyConfig: options.changed.connect(self.configure) def configure(self, options: moptions.Options, updated: typing.Any) -> None: - if "ignore_hosts" in updated: - self.check_ignore = HostMatcher(options.ignore_hosts) + if options.allow_hosts and options.ignore_hosts: + raise exceptions.OptionsError("--ignore-hosts and --allow-hosts are mutually " + "exclusive; please choose one.") + + if options.ignore_hosts: + self.check_filter = HostMatcher("ignore", options.ignore_hosts) + elif options.allow_hosts: + self.check_filter = HostMatcher("allow", options.allow_hosts) + else: + self.check_filter = HostMatcher(False) if "tcp_hosts" in updated: - self.check_tcp = HostMatcher(options.tcp_hosts) + self.check_tcp = HostMatcher("tcp", options.tcp_hosts) certstore_path = os.path.expanduser(options.confdir) if not os.path.exists(os.path.dirname(certstore_path)): diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index eae7f06d..42b61f4d 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -642,7 +642,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr try: layer() - except exceptions.Http2ZombieException as e: # pragma: no cover + except exceptions.Http2ZombieException: # pragma: no cover pass except exceptions.ProtocolException as e: # pragma: no cover self.log(repr(e), "info") diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 3f337a2a..096aae9f 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -464,11 +464,12 @@ class TlsLayer(base.Layer): def _find_cert(self): """ - This function determines the Common Name (CN) and Subject Alternative Names (SANs) + This function determines the Common Name (CN), Subject Alternative Names (SANs) and Organization Name our certificate should have and then fetches a matching cert from the certstore. """ host = None sans = set() + organization = None # In normal operation, the server address should always be known at this point. # However, we may just want to establish TLS so that we can send an error message to the client, @@ -488,6 +489,8 @@ class TlsLayer(base.Layer): if upstream_cert.cn: sans.add(host) host = upstream_cert.cn.decode("utf8").encode("idna") + if upstream_cert.organization: + organization = upstream_cert.organization # Also add SNI values. if self._client_hello.sni: sans.add(self._client_hello.sni.encode("idna")) @@ -498,4 +501,4 @@ class TlsLayer(base.Layer): # In other words, the Common Name is irrelevant then. if host: sans.add(host) - return self.config.certstore.get_cert(host, list(sans)) + return self.config.certstore.get_cert(host, list(sans), organization) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 0d1964a6..f5ac6a29 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -4,8 +4,9 @@ from OpenSSL import SSL import wsproto -from wsproto import events -from wsproto.connection import ConnectionType, WSConnection +from wsproto import events, WSConnection +from wsproto.connection import ConnectionType +from wsproto.events import AcceptConnection, CloseConnection, Message, Ping, Request from wsproto.extensions import PerMessageDeflate from mitmproxy import exceptions @@ -52,51 +53,52 @@ class WebSocketLayer(base.Layer): self.connections: dict[object, WSConnection] = {} - extensions = [] + client_extensions = [] + server_extensions = [] if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers: if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']: - extensions = [PerMessageDeflate()] - self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER, - extensions=extensions) - self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT, - host=handshake_flow.request.host, - resource=handshake_flow.request.path, - extensions=extensions) - if extensions: - for conn in self.connections.values(): - conn.extensions[0].finalize(conn, handshake_flow.response.headers['Sec-WebSocket-Extensions']) - - data = self.connections[self.server_conn].bytes_to_send() - self.connections[self.client_conn].receive_bytes(data) + client_extensions = [PerMessageDeflate()] + server_extensions = [PerMessageDeflate()] + self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER) + self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT) + + if client_extensions: + client_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + if server_extensions: + server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + + request = Request(extensions=client_extensions, host=handshake_flow.request.host, target=handshake_flow.request.path) + data = self.connections[self.server_conn].send(request) + self.connections[self.client_conn].receive_data(data) event = next(self.connections[self.client_conn].events()) - assert isinstance(event, events.ConnectionRequested) + assert isinstance(event, events.Request) - self.connections[self.client_conn].accept(event) - self.connections[self.server_conn].receive_bytes(self.connections[self.client_conn].bytes_to_send()) - assert isinstance(next(self.connections[self.server_conn].events()), events.ConnectionEstablished) + data = self.connections[self.client_conn].send(AcceptConnection(extensions=server_extensions)) + self.connections[self.server_conn].receive_data(data) + assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection) def _handle_event(self, event, source_conn, other_conn, is_server): - if isinstance(event, events.DataReceived): - return self._handle_data_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PingReceived): - return self._handle_ping_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PongReceived): - return self._handle_pong_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.ConnectionClosed): - return self._handle_connection_closed(event, source_conn, other_conn, is_server) + if isinstance(event, events.Message): + return self._handle_message(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Ping): + return self._handle_ping(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Pong): + return self._handle_pong(event, source_conn, other_conn, is_server) + elif isinstance(event, events.CloseConnection): + return self._handle_close_connection(event, source_conn, other_conn, is_server) # fail-safe for unhandled events return True # pragma: no cover - def _handle_data_received(self, event, source_conn, other_conn, is_server): + def _handle_message(self, event, source_conn, other_conn, is_server): fb = self.server_frame_buffer if is_server else self.client_frame_buffer fb.append(event.data) if event.message_finished: original_chunk_sizes = [len(f) for f in fb] - if isinstance(event, events.TextReceived): + if isinstance(event, events.TextMessage): message_type = wsproto.frame_protocol.Opcode.TEXT payload = ''.join(fb) else: @@ -127,19 +129,20 @@ class WebSocketLayer(base.Layer): yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False) for chunk, final in get_chunk(websocket_message.content): - self.connections[other_conn].send_data(chunk, final) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=chunk, message_finished=final)) + other_conn.send(data) if self.flow.stream: - self.connections[other_conn].send_data(event.data, event.message_finished) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=event.data, message_finished=event.message_finished)) + other_conn.send(data) return True - def _handle_ping_received(self, event, source_conn, other_conn, is_server): - # PING is automatically answered with a PONG by wsproto - self.connections[other_conn].ping() - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + def _handle_ping(self, event, source_conn, other_conn, is_server): + # Use event.response to create the approprate Pong response + data = self.connections[other_conn].send(Ping()) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) self.log( "Ping Received from {}".format("server" if is_server else "client"), "info", @@ -147,7 +150,7 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_pong_received(self, event, source_conn, other_conn, is_server): + def _handle_pong(self, event, source_conn, other_conn, is_server): self.log( "Pong Received from {}".format("server" if is_server else "client"), "info", @@ -155,14 +158,15 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_connection_closed(self, event, source_conn, other_conn, is_server): + def _handle_close_connection(self, event, source_conn, other_conn, is_server): self.flow.close_sender = "server" if is_server else "client" self.flow.close_code = event.code self.flow.close_reason = event.reason - self.connections[other_conn].close(event.code, event.reason) - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[other_conn].send(CloseConnection(code=event.code, reason=event.reason)) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) return False @@ -170,8 +174,7 @@ class WebSocketLayer(base.Layer): while True: try: payload = message_queue.get_nowait() - self.connections[endpoint].send_data(payload, final=True) - data = self.connections[endpoint].bytes_to_send() + data = self.connections[endpoint].send(Message(data=payload, message_finished=True)) endpoint.send(data) except queue.Empty: break @@ -197,8 +200,8 @@ class WebSocketLayer(base.Layer): is_server = (source_conn == self.server_conn) frame = websockets.Frame.from_file(source_conn.rfile) - self.connections[source_conn].receive_bytes(bytes(frame)) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[source_conn].receive_data(bytes(frame)) + source_conn.send(data) if close_received: return diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index eb0008cf..4805f874 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -48,17 +48,17 @@ class RootContext: raise exceptions.ProtocolException(str(e)) client_tls = tls.is_tls_record_magic(d) - # 1. check for --ignore - if self.config.check_ignore: - ignore = self.config.check_ignore(top_layer.server_conn.address) - if not ignore and client_tls: + # 1. check for filter + if self.config.check_filter: + is_filtered = self.config.check_filter(top_layer.server_conn.address) + if not is_filtered and client_tls: try: client_hello = tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: - ignore = self.config.check_ignore((client_hello.sni, 443)) - if ignore: + is_filtered = self.config.check_filter((client_hello.sni, 443)) + if is_filtered: return protocol.RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index 8baaa1c4..b95d73ab 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -87,7 +87,7 @@ def run( arg_check.check() sys.exit(1) try: - opts.confdir = args.confdir + opts.set(*args.setoptions, defer=True) optmanager.load_paths( opts, os.path.join(opts.confdir, OPTIONS_FILE_NAME), @@ -110,7 +110,6 @@ def run( if args.commands: master.commands.dump() sys.exit(0) - opts.set(*args.setoptions, defer=True) if extra: opts.update(**extra(args)) @@ -134,7 +133,7 @@ def run( except exceptions.OptionsError as e: print("%s: %s" % (sys.argv[0], e), file=sys.stderr) sys.exit(1) - except (KeyboardInterrupt, RuntimeError) as e: + except (KeyboardInterrupt, RuntimeError): pass return master diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..eb9147b3 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,7 +1,5 @@ import argparse -from mitmproxy.addons import core - def common_options(parser, opts): parser.add_argument( @@ -21,12 +19,6 @@ def common_options(parser, opts): help="Show all commands and their signatures", ) parser.add_argument( - "--confdir", - type=str, dest="confdir", default=core.CONF_DIR, - metavar="PATH", - help="Path to the mitmproxy config directory" - ) - parser.add_argument( "--set", type=str, dest="setoptions", default=[], action="append", @@ -65,6 +57,7 @@ def common_options(parser, opts): opts.make_parser(group, "listen_port", metavar="PORT", short="p") opts.make_parser(group, "server", short="n") opts.make_parser(group, "ignore_hosts", metavar="HOST") + opts.make_parser(group, "allow_hosts", metavar="HOST") opts.make_parser(group, "tcp_hosts", metavar="HOST") opts.make_parser(group, "upstream_auth", metavar="USER:PASS") opts.make_parser(group, "proxyauth", metavar="SPEC") @@ -85,6 +78,7 @@ def common_options(parser, opts): opts.make_parser(group, "server_replay", metavar="PATH", short="S") opts.make_parser(group, "server_replay_kill_extra") opts.make_parser(group, "server_replay_nopop") + opts.make_parser(group, "server_replay_refresh") # Replacements group = parser.add_argument_group("Replacements") diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index 3db03d3e..c738e349 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -34,4 +34,4 @@ class CommandExecutor: ret, ), valign="top" - )
\ No newline at end of file + ) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 72282015..43ab50cb 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,6 +1,10 @@ import platform import typing +import datetime +import time +import math from functools import lru_cache +from publicsuffix2 import get_sld, get_tld import urwid import urwid.util @@ -97,16 +101,180 @@ if urwid.util.detected_encoding and not IS_WSL: SYMBOL_MARK = u"\u25cf" SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" + SYMBOL_ELLIPSIS = u"\u2026" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" SYMBOL_MARK = "[m]" SYMBOL_UP = "^" SYMBOL_DOWN = " " + SYMBOL_ELLIPSIS = "~" + + +def fixlen(s, maxlen): + if len(s) <= maxlen: + return s.ljust(maxlen) + else: + return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS + + +def fixlen_r(s, maxlen): + if len(s) <= maxlen: + return s.rjust(maxlen) + else: + return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):] + + +class TruncatedText(urwid.Widget): + def __init__(self, text, attr, align='left'): + self.text = text + self.attr = attr + self.align = align + super(TruncatedText, self).__init__() + + def pack(self, size, focus=False): + return (len(self.text), 1) + + def rows(self, size, focus=False): + return 1 + + def render(self, size, focus=False): + text = self.text + attr = self.attr + if self.align == 'right': + text = text[::-1] + attr = attr[::-1] + + text_len = len(text) # TODO: unicode? + if size is not None and len(size) > 0: + width = size[0] + else: + width = text_len + + if width >= text_len: + remaining = width - text_len + if remaining > 0: + c_text = text + ' ' * remaining + c_attr = attr + [('text', remaining)] + else: + c_text = text + c_attr = attr + else: + visible_len = width - len(SYMBOL_ELLIPSIS) + visible_text = text[0:visible_len] + c_text = visible_text + SYMBOL_ELLIPSIS + c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) + + [('focus', len(SYMBOL_ELLIPSIS.encode()))]) + + if self.align == 'right': + c_text = c_text[::-1] + c_attr = c_attr[::-1] + + return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width) + + +def truncated_plain(text, attr, align='left'): + return TruncatedText(text, [(attr, len(text.encode()))], align) + + +# Work around https://github.com/urwid/urwid/pull/330 +def rle_append_beginning_modify(rle, a_r): + """ + Append (a, r) (unpacked from *a_r*) to BEGINNING of rle. + Merge with first run when possible + + MODIFIES rle parameter contents. Returns None. + """ + a, r = a_r + if not rle: + rle[:] = [(a, r)] + else: + al, run = rle[0] + if a == al: + rle[0] = (a, run + r) + else: + rle[0:0] = [(a, r)] + + +def colorize_host(host): + tld = get_tld(host) + sld = get_sld(host) + + attr = [] + + tld_size = len(tld) + sld_size = len(sld) - tld_size + + for letter in reversed(range(len(host))): + character = host[letter] + if tld_size > 0: + style = 'url_domain' + tld_size -= 1 + elif tld_size == 0: + style = 'text' + tld_size -= 1 + elif sld_size > 0: + sld_size -= 1 + style = 'url_extension' + else: + style = 'text' + rle_append_beginning_modify(attr, (style, len(character.encode()))) + return attr + + +def colorize_req(s): + path = s.split('?', 2)[0] + i_query = len(path) + i_last_slash = path.rfind('/') + i_ext = path[i_last_slash + 1:].rfind('.') + i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) + in_val = False + attr = [] + for i in range(len(s)): + c = s[i] + if ((i < i_query and c == '/') or + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): + a = 'url_punctuation' + elif i > i_query: + if in_val: + if c == '&': + in_val = False + a = 'url_punctuation' + else: + a = 'url_query_value' + else: + if c == '=': + in_val = True + a = 'url_punctuation' + else: + a = 'url_query_key' + elif i > i_ext: + a = 'url_extension' + elif i > i_last_slash: + a = 'url_filename' + else: + a = 'text' + urwid.util.rle_append_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_url(url): + parts = url.split('/', 3) + if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':': + return [('error', len(url))] # bad URL + schemes = { + 'http:': 'scheme_http', + 'https:': 'scheme_https', + } + return [ + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) @lru_cache(maxsize=800) -def raw_format_flow(f, flow): +def raw_format_list(f): f = dict(f) pile = [] req = [] @@ -126,8 +294,7 @@ def raw_format_flow(f, flow): if f["req_is_replay"]: req.append(fcol(SYMBOL_REPLAY, "replay")) - pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else '' - req.append(fcol(f["req_method"] + pushed, "method")) + req.append(fcol(f["req_method"], "method")) preamble = sum(i[1] for i in req) + len(req) - 1 @@ -140,8 +307,8 @@ def raw_format_flow(f, flow): url = f["req_url"] - if f["max_url_len"] and len(url) > f["max_url_len"]: - url = url[:f["max_url_len"]] + "…" + if f["cols"] and len(url) > f["cols"]: + url = url[:f["cols"]] + "…" if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): url += " " + f["req_http_version"] @@ -178,7 +345,8 @@ def raw_format_flow(f, flow): if f["resp_ctype"]: resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["roundtrip"], rc)) + pretty_duration = human.pretty_duration(f["duration"]) + resp.append(fcol(pretty_duration, rc)) elif f["err_msg"]: resp.append(fcol(SYMBOL_RETURN, "error")) @@ -194,48 +362,203 @@ def raw_format_flow(f, flow): return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): +@lru_cache(maxsize=800) +def raw_format_table(f): + f = dict(f) + pile = [] + req = [] + + cursor = [' ', 'focus'] + if f.get('resp_is_replay', False): + cursor[0] = SYMBOL_REPLAY + cursor[1] = 'replay' + if f['marked']: + if cursor[0] == ' ': + cursor[0] = SYMBOL_MARK + cursor[1] = 'mark' + if f['focus']: + cursor[0] = '>' + + req.append(fcol(*cursor)) + + if f["two_line"]: + req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) + pile.append(urwid.Columns(req, dividechars=1)) + + req = [] + req.append(fcol(' ', 'text')) + + if f["intercepted"] and not f["acked"]: + uc = "intercept" + elif "resp_code" in f or f["err_msg"] is not None: + uc = "highlight" + else: + uc = "title" + + if f["extended"]: + s = human.format_timestamp(f["req_timestamp"]) + else: + s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") + req.append(fcol(s, uc)) + + methods = { + 'GET': 'method_get', + 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' + } + uc = methods.get(f["req_method"], "method_other") + if f['extended']: + req.append(fcol(f["req_method"], uc)) + if f["req_promise"]: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + else: + if f["req_promise"]: + uc = 'method_http2_push' + req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) + + if f["two_line"]: + req.append(fcol(f["req_http_version"], 'text')) + else: + schemes = { + 'http': 'scheme_http', + 'https': 'scheme_https', + } + req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) + + req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) + req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) + + ret = (' ' * len(SYMBOL_RETURN), 'text') + status = ('', 'text') + content = ('', 'text') + size = ('', 'text') + duration = ('', 'text') + + if "resp_code" in f: + codes = { + 2: "code_200", + 3: "code_300", + 4: "code_400", + 5: "code_500", + } + ccol = codes.get(f["resp_code"] // 100, "code_other") + ret = (SYMBOL_RETURN, ccol) + status = (str(f["resp_code"]), ccol) + + if f["resp_len"] < 0: + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "content_none" + + if f["resp_len"] == -1: + contentdesc = "[content missing]" + else: + contentdesc = "[no content]" + content = (contentdesc, rc) + else: + if f["resp_ctype"]: + ctype = f["resp_ctype"].split(";")[0] + if ctype.endswith('/javascript'): + rc = 'content_script' + elif ctype.startswith('text/'): + rc = 'content_text' + elif (ctype.startswith('image/') or + ctype.startswith('video/') or + ctype.startswith('font/') or + "/x-font-" in ctype): + rc = 'content_media' + elif ctype.endswith('/json') or ctype.endswith('/xml'): + rc = 'content_data' + elif ctype.startswith('application/'): + rc = 'content_raw' + else: + rc = 'content_other' + content = (ctype, rc) + + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) + + size_str = human.pretty_size(f["resp_len"]) + if not f['extended']: + # shorten to 5 chars max + if len(size_str) > 5: + size_str = size_str[0:4].rstrip('.') + size_str[-1:] + size = (size_str, rc) + + if f['duration'] is not None: + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) + duration = (human.pretty_duration(f['duration']), rc) + + elif f["err_msg"]: + status = ('Err', 'error') + content = f["err_msg"], 'error' + + if f["two_line"]: + req.append(fcol(*ret)) + req.append(fcol(fixlen(status[0], 3), status[1])) + req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) + if f['extended']: + req.append(fcol(*size)) + else: + req.append(fcol(fixlen_r(size[0], 5), size[1])) + req.append(fcol(fixlen_r(duration[0], 5), duration[1])) + + pile.append(urwid.Columns(req, dividechars=1, min_width=15)) + + return urwid.Pile(pile) + + +def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'): acked = False if f.reply and f.reply.state == "committed": acked = True d = dict( focus=focus, extended=extended, - max_url_len=max_url_len, + two_line=extended or cols < 100, + cols=cols, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, req_is_replay=f.request.is_replay, req_method=f.request.method, + req_promise='h2-pushed-stream' in f.metadata, req_url=f.request.pretty_url if hostheader else f.request.url, + req_scheme=f.request.scheme, + req_host=f.request.pretty_host if hostheader else f.request.host, + req_path=f.request.path, req_http_version=f.request.http_version, err_msg=f.error.msg if f.error else None, marked=f.marked, ) if f.response: if f.response.raw_content: + content_len = len(f.response.raw_content) contentdesc = human.pretty_size(len(f.response.raw_content)) elif f.response.raw_content is None: + content_len = -1 contentdesc = "[content missing]" else: + content_len = -2 contentdesc = "[no content]" - duration = 0 + + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start - roundtrip = human.pretty_duration(duration) d.update(dict( resp_code=f.response.status_code, resp_reason=f.response.reason, resp_is_replay=f.response.is_replay, + resp_len=content_len, + resp_ctype=f.response.headers.get("content-type"), resp_clen=contentdesc, - roundtrip=roundtrip, + duration=duration, )) - t = f.response.headers.get("content-type") - if t: - d["resp_ctype"] = t.split(";")[0] - else: - d["resp_ctype"] = "" - - return raw_format_flow(tuple(sorted(d.items())), f) + if ((layout == 'default' and cols < 100) or layout == "list"): + return raw_format_list(tuple(sorted(d.items()))) + else: + return raw_format_table(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..b6602413 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -37,6 +37,12 @@ console_layouts = [ "horizontal", ] +console_flowlist_layout = [ + "default", + "table", + "list" +] + class UnsupportedLog: """ @@ -114,6 +120,13 @@ class ConsoleAddon: "Console mouse interaction." ) + loader.add_option( + "console_flowlist_layout", + str, "default", + "Set the flowlist layout", + choices=sorted(console_flowlist_layout) + ) + @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: """ @@ -428,7 +441,12 @@ class ConsoleAddon: message.content = c.rstrip(b"\n") elif part == "set-cookies": self.master.switch_view("edit_focus_setcookies") - elif part in ["url", "method", "status_code", "reason"]: + elif part == "url": + url = flow.request.url.encode() + edited_url = self.master.spawn_editor(url) + url = edited_url.rstrip(b"\n") + flow.request.url = url.decode() + elif part in ["method", "status_code", "reason"]: self.master.commands.execute( "console.command flow.set @focus %s " % part ) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e947a582..9650c0d3 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -18,7 +18,8 @@ class FlowItem(urwid.WidgetWrap): self.flow, self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) def selectable(self): @@ -84,6 +85,10 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): ) -> None: self.master: "mitmproxy.tools.console.master.ConsoleMaster" = master super().__init__(FlowListWalker(master)) + self.master.options.subscribe( + self.set_flowlist_layout, + ["console_flowlist_layout"] + ) def keypress(self, size, key): if key == "m_start": @@ -96,3 +101,6 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): def view_changed(self): self.body.view_changed() + + def set_flowlist_layout(self, opts, updated): + self.master.ui.clear() diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 87671c3b..807c9714 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -6,6 +6,7 @@ from typing import Optional, Union # noqa import urwid from mitmproxy import contentviews +from mitmproxy import ctx from mitmproxy import http from mitmproxy.tools.console import common from mitmproxy.tools.console import layoutwidget @@ -37,7 +38,8 @@ class FlowViewHeader(urwid.WidgetWrap): False, extended=True, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) else: self._w = urwid.Pile([]) @@ -102,7 +104,7 @@ class FlowDetails(tabs.Tabs): if full == "true": limit = sys.maxsize else: - limit = contentviews.VIEW_CUTOFF + limit = ctx.options.content_view_lines_cutoff flow_modify_cache_invalidation = hash(( message.raw_content, diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 1b4b9ac6..fb4e0051 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -91,9 +91,9 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): ) ) examples = [ - ("google\.com", "Url containing \"google.com"), - ("~q ~b test", "Requests where body contains \"test\""), - ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), + (r"google\.com", r"Url containing \"google.com"), + ("~q ~b test", r"Requests where body contains \"test\""), + (r"!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] text.extend( common.format_keyvals(examples, indent=4) diff --git a/mitmproxy/tools/console/keymap.py b/mitmproxy/tools/console/keymap.py index d22420bf..01ec9a0a 100644 --- a/mitmproxy/tools/console/keymap.py +++ b/mitmproxy/tools/console/keymap.py @@ -199,11 +199,14 @@ class KeymapConfig: "Error reading %s: %s" % (p, e) ) from e for v in vals: + user_ctxs = v.get("ctx", ["global"]) try: + km._check_contexts(user_ctxs) + km.remove(v["key"], Contexts) km.add( key = v["key"], command = v["cmd"], - contexts = v.get("ctx", ["global"]), + contexts = user_ctxs, help = v.get("help", None), ) except ValueError as e: diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 1bc79e5c..dd15a2f5 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -128,7 +128,7 @@ class ConsoleMaster(master.Master): subprocess.call(cmd) except: signals.status_message.send( - message="Can't start editor: %s" % " ".join(c) + message="Can't start editor: %s" % c ) else: with open(name, "r" if text else "rb") as f: diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 7930c4a3..4eee7692 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,7 +22,12 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'focus', + 'method', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', + 'scheme_http', 'scheme_https', 'scheme_other', + 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', + 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', + 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', @@ -36,6 +41,7 @@ class Palette: # Commander 'commander_command', 'commander_invalid', 'commander_hint' ] + _fields.extend(['gradient_%02d' % i for i in range(100)]) high: typing.Mapping[str, typing.Sequence[str]] = None def palette(self, transparent): @@ -68,6 +74,27 @@ class Palette: return l +def gen_gradient(palette, cols): + for i in range(100): + palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default') + + +def gen_rgb_gradient(palette, cols): + parts = len(cols) - 1 + for i in range(100): + p = i / 100 + idx = int(p * parts) + t0 = cols[idx] + t1 = cols[idx + 1] + pp = p * parts % 1 + t = ( + round(t0[0] + (t1[0] - t0[0]) * pp), + round(t0[1] + (t1[1] - t0[1]) * pp), + round(t0[2] + (t1[2] - t0[2]) * pp), + ) + palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default') + + class LowDark(Palette): """ @@ -95,6 +122,33 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), + method_other = ('dark magenta', 'default'), + method_http2_push = ('dark gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('dark green', 'default'), + scheme_other = ('dark magenta', 'default'), + + url_punctuation = ('light gray', 'default'), + url_domain = ('white', 'default'), + url_filename = ('dark cyan', 'default'), + url_extension = ('light gray', 'default'), + url_query_key = ('white', 'default'), + url_query_value = ('light gray', 'default'), + + content_none = ('dark gray', 'default'), + content_text = ('light gray', 'default'), + content_script = ('dark green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('dark red', 'default'), + content_other = ('dark magenta', 'default'), + focus = ('yellow', 'default'), code_200 = ('dark green', 'default'), @@ -127,6 +181,7 @@ class LowDark(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('dark gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Dark(LowDark): @@ -167,6 +222,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -198,6 +280,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -256,7 +339,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -311,9 +414,28 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections - method = (sol_cyan, 'default'), focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = (sol_base01, 'default'), + + url_punctuation = ('h242', 'default'), + url_domain = ('h252', 'default'), + url_filename = ('h132', 'default'), + url_extension = ('h96', 'default'), + url_query_key = ('h37', 'default'), + url_query_value = ('h30', 'default'), + + content_none = (sol_base01, 'default'), + content_text = (sol_base1, 'default'), + content_media = (sol_blue, 'default'), + code_200 = (sol_green, 'default'), code_300 = (sol_blue, 'default'), code_400 = (sol_orange, 'default',), @@ -342,6 +464,7 @@ class SolarizedDark(LowDark): commander_invalid = (sol_orange, 'default'), commander_hint = (sol_base00, 'default'), ) + gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)]) DEFAULT = "dark" diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 2d32f487..56f0674f 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -215,6 +215,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "I")) r.append("gnore:%d]" % len(self.master.options.ignore_hosts)) + elif self.master.options.allow_hosts: + r.append("[") + r.append(("heading_key", "A")) + r.append("llow:%d]" % len(self.master.options.allow_hosts)) if self.master.options.tcp_hosts: r.append("[") r.append(("heading_key", "T")) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index b72e0d77..6bfce34e 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -370,7 +370,7 @@ class FlowContent(RequestHandler): original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: - filename = re.search('filename=([-\w" .()]+)', original_cd) + filename = re.search(r'filename=([-\w" .()]+)', original_cd) if filename: filename = filename.group(1) if not filename: @@ -432,7 +432,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "upstream_cert", + "intercept", "showhost", "upstream_cert", "ssl_insecure", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 71d1c54c..388c765f 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -169,7 +169,7 @@ def split_special_areas( >>> split_special_areas( >>> "test /* don't modify me */ foo", - >>> [r"/\*[\s\S]*?\*/"]) # (regex matching comments) + >>> [r"/\\*[\\s\\S]*?\\*/"]) # (regex matching comments) ["test ", "/* don't modify me */", " foo"] "".join(split_special_areas(x, ...)) == x always holds true. @@ -201,7 +201,7 @@ def escape_special_areas( >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"]) >>> print(x) if (true) { console.log('�}'); } - >>> x = re.sub(r"\s*{\s*", " {\n ", x) + >>> x = re.sub(r"\\s*{\\s*", " {\n ", x) >>> x = unescape_special_areas(x) >>> print(x) if (true) { diff --git a/release/cibuild.py b/release/cibuild.py index ba17ced2..82060639 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -279,7 +279,7 @@ class BuildEnviron: if self.tag.startswith("v"): try: parver.Version.parse(self.tag[1:], strict=True) - except parver.ParseError as e: + except parver.ParseError: return self.tag return self.tag[1:] return self.tag diff --git a/release/docker/Dockerfile b/release/docker/Dockerfile index 7c9a2421..258bccf5 100644 --- a/release/docker/Dockerfile +++ b/release/docker/Dockerfile @@ -21,7 +21,8 @@ RUN addgroup -S mitmproxy && adduser -S -G mitmproxy mitmproxy \ openssl-dev \ python3 \ python3-dev \ - && python3 -m ensurepip \ + && python3 -m ensurepip --upgrade \ + && pip3 install -U pip \ && LDFLAGS=-L/lib pip3 install -U /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} \ && apk del --purge \ git \ diff --git a/release/docker/DockerfileARMv7 b/release/docker/DockerfileARMv7 index 6e04b7ad..40f10ede 100644 --- a/release/docker/DockerfileARMv7 +++ b/release/docker/DockerfileARMv7 @@ -23,7 +23,8 @@ RUN addgroup -S mitmproxy && adduser -S -G mitmproxy mitmproxy \ openssl-dev \ python3 \ python3-dev \ - && python3 -m ensurepip \ + && python3 -m ensurepip --upgrade \ + && pip3 install -U pip \ && LDFLAGS=-L/lib pip3 install -U /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} \ && apk del --purge \ git \ @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,C901,W503,W292,E722,E741 +ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp @@ -59,7 +59,6 @@ exclude = mitmproxy/net/http/headers.py mitmproxy/net/http/message.py mitmproxy/net/http/multipart.py - mitmproxy/net/http/url.py mitmproxy/net/tcp.py mitmproxy/net/tls.py mitmproxy/options.py @@ -62,25 +62,27 @@ setup( # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "blinker>=1.4, <1.5", - "brotlipy>=0.7.0,<0.8", - "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! + "Brotli>=1.0,<1.1", + "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=6.2, <7", - "cryptography>=2.1.4,<2.4", + "cryptography>=2.1.4,<2.5", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", - "ldap3>=2.5,<2.6", + "ldap3>=2.6.1,<2.7", "passlib>=1.6.5, <1.8", - "protobuf>=3.6.0, <3.7", + "protobuf>=3.6.0, <3.10", "pyasn1>=0.3.1,<0.5", - "pyOpenSSL>=17.5,<18.1", - "pyparsing>=2.1.3, <2.3", - "pyperclip>=1.6.0, <1.7", - "ruamel.yaml>=0.13.2, <0.15.55", + "pyOpenSSL>=19.0.0,<20", + "pyparsing>=2.4.2,<2.5", + "pyperclip>=1.6.0,<1.8", + "ruamel.yaml>=0.15,<0.16", "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", - "wsproto>=0.11.0,<0.12.0", + "wsproto>=0.14.0,<0.15.0", + "publicsuffix2~=2.20", + "zstandard>=0.11.0,<0.13.0", ], extras_require={ ':sys_platform == "win32"': [ @@ -88,8 +90,8 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5, <3.6", - "Flask>=1.0,<1.1", + "flake8>=3.5,<=3.7.8", + "Flask>=1.0,<1.2", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", "pytest-asyncio>=0.8", @@ -97,9 +99,9 @@ setup( "pytest-faulthandler>=1.3.1,<2", "pytest-timeout>=1.2.1,<2", "pytest-xdist>=1.22,<2", - "pytest>=3.3,<4", + "pytest>=4.0,<5", "requests>=2.9.1, <3", - "tox>=3.0,<3.2", + "tox>=3.5,<3.6", "rstcheck>=2.2, <4.0", ], 'examples': [ diff --git a/test/bench/benchmark.py b/test/bench/benchmark.py index 84ec6005..076ad6c9 100644 --- a/test/bench/benchmark.py +++ b/test/bench/benchmark.py @@ -31,7 +31,8 @@ class Benchmark: stdout=asyncio.subprocess.PIPE ) stdout, _ = await traf.communicate() - open(ctx.options.benchmark_save_path + ".bench", mode="wb").write(stdout) + with open(ctx.options.benchmark_save_path + ".bench", mode="wb") as f: + f.write(stdout) ctx.log.error("Proxy saw %s requests, %s responses" % (self.reqs, self.resps)) ctx.log.error(stdout.decode("ascii")) backend.kill() diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index c24801e4..7a41c7b9 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -32,37 +32,50 @@ def test_configure(): def test_simple(): sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=1) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=1) d.error(tflow.tflow(err=True)) assert sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert "<<" in sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) d.response(tflow.tflow(err=True)) assert "<<" in sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) flow = tflow.tflow() @@ -75,6 +88,8 @@ def test_simple(): d.response(flow) assert sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) flow = tflow.tflow(resp=tutils.tresp(content=b"{")) @@ -83,6 +98,8 @@ def test_simple(): d.response(flow) assert sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) ctx.configure(d, flow_detail=4) flow = tflow.tflow() @@ -92,6 +109,8 @@ def test_simple(): d.response(flow) assert "content missing" in sio.getvalue() sio.truncate(0) + assert not sio_err.getvalue() + sio_err.truncate(0) def test_echo_body(): @@ -100,7 +119,8 @@ def test_echo_body(): f.response.content = b"foo bar voing\n" * 100 sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3) d._echo_message(f.response) @@ -110,7 +130,8 @@ def test_echo_body(): def test_echo_request_line(): sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) @@ -146,7 +167,8 @@ class TestContentView: with mock.patch("mitmproxy.contentviews.auto.ViewAuto.__call__") as va: va.side_effect = exceptions.ContentViewException("") sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=4) d.response(tflow.tflow()) @@ -155,7 +177,8 @@ class TestContentView: def test_tcp(): sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow() @@ -165,12 +188,13 @@ def test_tcp(): f = tflow.ttcpflow(client_conn=True, err=True) d.tcp_error(f) - assert "Error in TCP" in sio.getvalue() + assert "Error in TCP" in sio_err.getvalue() def test_websocket(): sio = io.StringIO() - d = dumper.Dumper(sio) + sio_err = io.StringIO() + d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.twebsocketflow() @@ -183,4 +207,4 @@ def test_websocket(): f = tflow.twebsocketflow(client_conn=True, err=True) d.websocket_error(f) - assert "Error in WebSocket" in sio.getvalue() + assert "Error in WebSocket" in sio_err.getvalue() diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index 3d28d8b7..94e18cdb 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -30,8 +30,8 @@ def data(): @pytest.fixture -def corrupt_data(): - f = data() +def corrupt_data(data): + f = io.BytesIO(data.getvalue()) f.seek(0, io.SEEK_END) f.write(b"qibble") f.seek(0) diff --git a/test/mitmproxy/addons/test_session.py b/test/mitmproxy/addons/test_session.py index 20feb69d..97351426 100644 --- a/test/mitmproxy/addons/test_session.py +++ b/test/mitmproxy/addons/test_session.py @@ -68,7 +68,8 @@ class TestSession: os.remove(path) con = sqlite3.connect(path) script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() + with open(script_path) as f: + qry = f.read() with con: con.executescript(qry) blob = b'blob_of_data' diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 976c14b7..f5088a68 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -471,7 +471,7 @@ def test_focus(): v = view.View() v.add([tft()]) f = view.Focus(v) - assert f.index is 0 + assert f.index == 0 assert f.flow is v[0] # Start empty diff --git a/test/mitmproxy/contentviews/test_base.py b/test/mitmproxy/contentviews/test_base.py index c94d8be2..cd879bfd 100644 --- a/test/mitmproxy/contentviews/test_base.py +++ b/test/mitmproxy/contentviews/test_base.py @@ -15,3 +15,18 @@ def test_format_dict(): f_d = base.format_dict(d) with pytest.raises(StopIteration): next(f_d) + + +def test_format_pairs(): + d = [("a", "c"), ("b", "d")] + f_d = base.format_pairs(d) + assert next(f_d) + + d = [("abc", "")] + f_d = base.format_pairs(d) + assert next(f_d) + + d = [] + f_d = base.format_pairs(d) + with pytest.raises(StopIteration): + next(f_d) diff --git a/test/mitmproxy/contentviews/test_query.py b/test/mitmproxy/contentviews/test_query.py index 741b23f1..1ae1b3ac 100644 --- a/test/mitmproxy/contentviews/test_query.py +++ b/test/mitmproxy/contentviews/test_query.py @@ -6,8 +6,8 @@ from . import full_eval def test_view_query(): d = "" v = full_eval(query.ViewQuery()) - f = v(d, query=multidict.MultiDict([("foo", "bar")])) + f = v(d, query=multidict.MultiDict([("foo", "bar"), ("foo", "baz")])) assert f[0] == "Query" - assert f[1] == [[("header", "foo: "), ("text", "bar")]] + assert f[1] == [[("header", "foo: "), ("text", "bar")], [("header", "foo: "), ("text", "baz")]] assert v(d) == ("Query", []) diff --git a/test/mitmproxy/coretypes/test_basethread.py b/test/mitmproxy/coretypes/test_basethread.py index 4a383fea..6b0ae154 100644 --- a/test/mitmproxy/coretypes/test_basethread.py +++ b/test/mitmproxy/coretypes/test_basethread.py @@ -4,4 +4,4 @@ from mitmproxy.coretypes import basethread def test_basethread(): t = basethread.BaseThread('foobar') - assert re.match('foobar - age: \d+s', t._threadinfo()) + assert re.match(r'foobar - age: \d+s', t._threadinfo()) diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 74233cca..06cfe1d3 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -27,7 +27,7 @@ cookie_pairs = [ [["one", "uno"], ["two", "due"]] ], [ - 'one="uno"; two="\due"', + 'one="uno"; two="\\due"', [["one", "uno"], ["two", "due"]] ], [ @@ -70,7 +70,7 @@ def test_read_key(): def test_read_quoted_string(): tokens = [ [('"foo" x', 0), ("foo", 5)], - [('"f\oo" x', 0), ("foo", 6)], + [('"f\\oo" x', 0), ("foo", 6)], [(r'"f\\o" x', 0), (r"f\o", 6)], [(r'"f\\" x', 0), (r"f" + '\\', 5)], [('"fo\\\"" x', 0), ("fo\"", 6)], diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index 8dac12cb..7f768f39 100644 --- a/test/mitmproxy/net/http/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -19,6 +19,7 @@ def test_identity(encoder): 'gzip', 'br', 'deflate', + 'zstd', ]) def test_encoders(encoder): """ diff --git a/test/mitmproxy/net/http/test_headers.py b/test/mitmproxy/net/http/test_headers.py index e8eb31d3..8fc8b027 100644 --- a/test/mitmproxy/net/http/test_headers.py +++ b/test/mitmproxy/net/http/test_headers.py @@ -75,7 +75,7 @@ class TestHeaders: def test_replace_multi(self): headers = self._2host() - headers.replace(r"Host: example\.com", r"Host: example.de") + headers.replace(r"Host: example.com", r"Host: example.de") assert headers.get_all("Host") == ["example.de", "example.org"] def test_replace_remove_spacer(self): diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py index 512f3199..7ad7890c 100644 --- a/test/mitmproxy/net/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -229,6 +229,16 @@ class TestMessageText: r.headers["content-type"] = "application/json" assert r.text == u'"ü"' + def test_guess_meta_charset(self): + r = tutils.tresp(content=b'<meta http-equiv="content-type" ' + b'content="text/html;charset=gb2312">\xe6\x98\x8e\xe4\xbc\xaf') + # "鏄庝集" is decoded form of \xe6\x98\x8e\xe4\xbc\xaf in gb18030 + assert u"鏄庝集" in r.text + + def test_guess_latin_1(self): + r = tutils.tresp(content=b"\xF0\xE2") + assert r.text == u"ðâ" + def test_none(self): r = tutils.tresp(content=None) assert r.text is None diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index f3470384..27c16be6 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -148,7 +148,7 @@ class TestResponseUtils: def test_refresh(self): r = tresp() n = time.time() - r.headers["date"] = email.utils.formatdate(n) + r.headers["date"] = email.utils.formatdate(n, usegmt=True) pre = r.headers["date"] r.refresh(946681202) assert pre == r.headers["date"] diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index ecf8e896..48277859 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -49,6 +49,17 @@ def test_parse(): url.parse('http://lo[calhost') +def test_ascii_check(): + + test_url = "https://xyz.tax-edu.net?flag=selectCourse&lc_id=42825&lc_name=茅莽莽猫氓猫氓".encode() + scheme, host, port, full_path = url.parse(test_url) + assert scheme == b'https' + assert host == b'xyz.tax-edu.net' + assert port == 443 + assert full_path == b'/?flag%3DselectCourse%26lc_id%3D42825%26lc_name%3D%E8%8C%85%E8%8E%BD%E8%8E' \ + b'%BD%E7%8C%AB%E6%B0%93%E7%8C%AB%E6%B0%93' + + @pytest.mark.skipif(sys.version_info < (3, 6), reason='requires Python 3.6 or higher') def test_parse_port_range(): # Port out of range @@ -61,6 +72,7 @@ def test_unparse(): assert url.unparse("http", "foo.com", 80, "/bar") == "http://foo.com/bar" assert url.unparse("https", "foo.com", 80, "") == "https://foo.com:80" assert url.unparse("https", "foo.com", 443, "") == "https://foo.com" + assert url.unparse("https", "foo.com", 443, "*") == "https://foo.com" # We ignore the byte 126: '~' because of an incompatibility in Python 3.6 and 3.7 @@ -131,3 +143,7 @@ def test_unquote(): assert url.unquote("foo") == "foo" assert url.unquote("foo%20bar") == "foo bar" assert url.unquote(surrogates_quoted) == surrogates + + +def test_hostport(): + assert url.hostport(b"https", b"foo.com", 8080) == b"foo.com:8080" diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index b6bb7cc1..22a306dc 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -102,7 +102,7 @@ class TestServerBind(tservers.ServerTestBase): # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. peername = self.connection.getpeername() - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) port = peername[1] self.wfile.write(str((address, port)).encode()) diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 1da031c6..1319d1a9 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -17,3 +17,12 @@ class TestProxyConfig: opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) + + def test_cannot_set_both_allow_and_filter_options(self): + opts = options.Options() + opts.ignore_hosts = ["foo"] + opts.allow_hosts = ["bar"] + with pytest.raises(exceptions.OptionsError, match="--ignore-hosts and --allow-hosts are " + "mutually exclusive; please choose " + "one."): + ProxyConfig(opts) diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 01ab068d..b5852d60 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -78,6 +78,16 @@ class TcpMixin: self.options.ignore_hosts = self._ignore_backup del self._ignore_backup + def _allow_on(self): + assert not hasattr(self, "_allow_backup") + self._allow_backup = self.options.allow_hosts + self.options.allow_hosts = ["(127.0.0.1|None):\\d+"] + self.options.allow_hosts + + def _allow_off(self): + assert hasattr(self, "_allow_backup") + self.options.allow_hosts = self._allow_backup + del self._allow_backup + def test_ignore(self): n = self.pathod("304") self._ignore_on() @@ -111,6 +121,40 @@ class TcpMixin: self._ignore_off() + def test_allow(self): + n = self.pathod("304") + self._allow_on() + i = self.pathod("305") + i2 = self.pathod("306") + self._allow_off() + + assert n.status_code == 304 + assert i.status_code == 305 + assert i2.status_code == 306 + + assert any(f.response.status_code == 304 for f in self.master.state.flows) + assert any(f.response.status_code == 305 for f in self.master.state.flows) + assert any(f.response.status_code == 306 for f in self.master.state.flows) + + # Test that we get the original SSL cert + if self.ssl: + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert + assert i_cert != n_cert + + # Test Non-HTTP traffic + spec = "200:i0,@100:d0" # this results in just 100 random bytes + # mitmproxy responds with bad gateway + assert self.pathod(spec).status_code == 502 + self._allow_on() + + self.pathod(spec) # pathoc parses answer as HTTP + + self._allow_off() + def _tcpproxy_on(self): assert not hasattr(self, "_tcpproxy_backup") self._tcpproxy_backup = self.options.tcp_hosts @@ -852,10 +896,12 @@ class TestUpstreamProxySSL( def _host_pattern_on(self, attr): """ - Updates config.check_tcp or check_ignore, depending on attr. + Updates config.check_tcp or check_filter, depending on attr. """ assert not hasattr(self, "_ignore_%s_backup" % attr) backup = [] + handle = attr + attr = "filter" if attr in ["allow", "ignore"] else attr for proxy in self.chain: old_matcher = getattr( proxy.tmaster.server.config, @@ -865,12 +911,13 @@ class TestUpstreamProxySSL( setattr( proxy.tmaster.server.config, "check_%s" % attr, - HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns) + HostMatcher(handle, [".+:%s" % self.server.port] + old_matcher.patterns) ) setattr(self, "_ignore_%s_backup" % attr, backup) def _host_pattern_off(self, attr): + attr = "filter" if attr in ["allow", "ignore"] else attr backup = getattr(self, "_ignore_%s_backup" % attr) for proxy in reversed(self.chain): setattr( diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 3ec58760..70d41511 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest): assert await tctx.master.await_log("decorator not supported") def test_concurrent_class(self, tdata): - with taddons.context() as tctx: - sc = tctx.script( - tdata.path( - "mitmproxy/data/addonscripts/concurrent_decorator_class.py" - ) + with taddons.context() as tctx: + sc = tctx.script( + tdata.path( + "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) - f1, f2 = tflow.tflow(), tflow.tflow() - tctx.cycle(sc, f1) - tctx.cycle(sc, f2) - start = time.time() - while time.time() - start < 5: - if f1.reply.state == f2.reply.state == "committed": - return - raise ValueError("Script never acked") + ) + f1, f2 = tflow.tflow(), tflow.tflow() + tctx.cycle(sc, f1) + tctx.cycle(sc, f2) + start = time.time() + while time.time() - start < 5: + if f1.reply.state == f2.reply.state == "committed": + return + raise ValueError("Script never acked") diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 12d3dc96..b8ad1d36 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -1,5 +1,6 @@ import os from mitmproxy import certs +from ..conftest import skip_windows # class TestDNTree: # def test_simple(self): @@ -111,6 +112,14 @@ class TestCertStore: certs.CertStore.load_dhparam(filename) assert os.path.exists(filename) + @skip_windows + def test_umask_secret(self, tmpdir): + filename = str(tmpdir.join("secret")) + with certs.CertStore.umask_secret(), open(filename, "wb"): + pass + # TODO: How do we actually attempt to read that file as another user? + assert os.stat(filename).st_mode & 0o77 == 0 + class TestDummyCert: @@ -120,18 +129,22 @@ class TestDummyCert: ca.default_privatekey, ca.default_ca, b"foo.com", - [b"one.com", b"two.com", b"*.three.com", b"127.0.0.1"] + [b"one.com", b"two.com", b"*.three.com", b"127.0.0.1"], + b"Foo Ltd." ) assert r.cn == b"foo.com" assert r.altnames == [b'one.com', b'two.com', b'*.three.com'] + assert r.organization == b"Foo Ltd." r = certs.dummy_cert( ca.default_privatekey, ca.default_ca, None, - [] + [], + None ) assert r.cn is None + assert r.organization is None assert r.altnames == [] @@ -143,6 +156,7 @@ class TestCert: c1 = certs.Cert.from_pem(d) assert c1.cn == b"google.com" assert len(c1.altnames) == 436 + assert c1.organization == b"Google Inc" with open(tdata.path("mitmproxy/net/data/text_cert_2"), "rb") as f: d = f.read() diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py index 4eb37d81..d53cec7d 100644 --- a/test/mitmproxy/test_flowfilter.py +++ b/test/mitmproxy/test_flowfilter.py @@ -28,6 +28,9 @@ class TestParsing: self._dump(p) assert len(p.lst) == 2 + def test_non_ascii(self): + assert flowfilter.parse("~s шгн") + def test_naked_url(self): a = flowfilter.parse("foobar ~h rex") assert a.lst[0].expr == "foobar" @@ -173,10 +176,30 @@ class TestMatchingHTTPFlow: assert not self.q("~bq message", q) assert not self.q("~bq message", s) + s.response.text = 'яч' # Cyrillic + assert self.q("~bs яч", s) + s.response.text = '测试' # Chinese + assert self.q('~bs 测试', s) + s.response.text = 'ॐ' # Hindi + assert self.q('~bs ॐ', s) + s.response.text = 'لله' # Arabic + assert self.q('~bs لله', s) + s.response.text = 'θεός' # Greek + assert self.q('~bs θεός', s) + s.response.text = 'לוהים' # Hebrew + assert self.q('~bs לוהים', s) + s.response.text = '神' # Japanese + assert self.q('~bs 神', s) + s.response.text = '하나님' # Korean + assert self.q('~bs 하나님', s) + s.response.text = 'Äÿ' # Latin + assert self.q('~bs Äÿ', s) + assert not self.q("~bs nomatch", s) assert not self.q("~bs content", q) assert not self.q("~bs content", s) assert not self.q("~bs message", q) + s.response.text = 'message' assert self.q("~bs message", s) def test_body(self): diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 49e61e25..8a299d8e 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -49,7 +49,7 @@ class TestHTTPRequest: r.path = "path/foo" r.headers["Foo"] = "fOo" r.content = b"afoob" - assert r.replace("foo(?i)", "boo") == 4 + assert r.replace("(?i)foo", "boo") == 4 assert r.path == "path/boo" assert b"foo" not in r.content assert r.headers["boo"] == "boo" @@ -82,7 +82,7 @@ class TestHTTPResponse: r = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) r.headers["Foo"] = "fOo" r.content = b"afoob" - assert r.replace("foo(?i)", "boo") == 3 + assert r.replace("(?i)foo", "boo") == 3 assert b"foo" not in r.content assert r.headers["boo"] == "boo" diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 00086c4b..c8cf6c33 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,5 @@ import argparse +import platform from unittest import mock import pytest @@ -52,8 +53,11 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows + @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], + reason='Skipping due to macOS Mojave') def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows + # binding to 0.0.0.0:1 works without special permissions on Windows and + # macOS Mojave conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) diff --git a/test/mitmproxy/tools/console/test_keymap.py b/test/mitmproxy/tools/console/test_keymap.py index 3e6f7c2e..0d6f9e88 100644 --- a/test/mitmproxy/tools/console/test_keymap.py +++ b/test/mitmproxy/tools/console/test_keymap.py @@ -117,6 +117,21 @@ def test_load_path(tmpdir): kmc.load_path(km, dst) assert(km.get("chooser", "key1")) + km.add("key123", "str", ["flowlist", "flowview"]) + with open(dst, 'w') as f: + f.write( + """ + - key: key123 + ctx: [options] + cmd: foo + """ + ) + kmc.load_path(km, dst) + for b in km.bindings: + if b.key == "key123": + assert b.contexts == ["options"] + break + def test_parse(): kmc = keymap.KeymapConfig() @@ -44,8 +44,8 @@ commands = passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER deps = -rrequirements.txt - pyinstaller==3.3.1 - twine==1.11.0 + pyinstaller==3.5 + twine==2.0.0 awscli commands = mitmdump --version diff --git a/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowColumnsSpec.js.snap b/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowColumnsSpec.js.snap index d6946507..98ba8a10 100644 --- a/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowColumnsSpec.js.snap +++ b/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowColumnsSpec.js.snap @@ -109,6 +109,11 @@ exports[`FlowColumns Components should render SizeColumn 1`] = ` exports[`FlowColumns Components should render StatusColumn 1`] = ` <td className="col-status" + style={ + Object { + "color": "darkred", + } + } /> `; diff --git a/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowRowSpec.js.snap b/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowRowSpec.js.snap index b065b746..786488a0 100644 --- a/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowRowSpec.js.snap +++ b/web/src/js/__tests__/components/FlowTable/__snapshots__/FlowRowSpec.js.snap @@ -30,6 +30,11 @@ exports[`FlowRow Component should render correctly 1`] = ` </td> <td className="col-status" + style={ + Object { + "color": "darkgreen", + } + } > 200 </td> diff --git a/web/src/js/components/FlowTable/FlowColumns.jsx b/web/src/js/components/FlowTable/FlowColumns.jsx index e60ed487..07b54a32 100644 --- a/web/src/js/components/FlowTable/FlowColumns.jsx +++ b/web/src/js/components/FlowTable/FlowColumns.jsx @@ -90,8 +90,26 @@ MethodColumn.headerClass = 'col-method' MethodColumn.headerName = 'Method' export function StatusColumn({ flow }) { + let color = 'darkred'; + + if (flow.response !== null && 100 <= flow.response.status_code && flow.response.status_code < 200) { + color = 'green' + } + else if (flow.response !== null && 200 <= flow.response.status_code && flow.response.status_code < 300) { + color = 'darkgreen' + } + else if (flow.response !== null && 300 <= flow.response.status_code && flow.response.status_code < 400) { + color = 'lightblue' + } + else if (flow.response !== null && 400 <= flow.response.status_code && flow.response.status_code < 500) { + color = 'lightred' + } + else if (flow.response !== null && 500 <= flow.response.status_code && flow.response.status_code < 600) { + color = 'lightred' + } + return ( - <td className="col-status">{flow.response && flow.response.status_code}</td> + <td className="col-status" style={{color: color}}>{flow.response && flow.response.status_code}</td> ) } diff --git a/web/src/js/ducks/ui/keyboard.js b/web/src/js/ducks/ui/keyboard.js index ed4dbba5..007d24db 100644 --- a/web/src/js/ducks/ui/keyboard.js +++ b/web/src/js/ducks/ui/keyboard.js @@ -6,7 +6,7 @@ import * as modalActions from "./modal" export function onKeyDown(e) { //console.debug("onKeyDown", e) - if (e.ctrlKey) { + if (e.ctrlKey || e.metaKey) { return () => { } } |