diff options
| -rw-r--r-- | .travis.yml | 1 | ||||
| -rw-r--r-- | docs/src/content/concepts-certificates.md | 3 | ||||
| -rw-r--r-- | examples/complex/har_dump.py | 5 | ||||
| -rw-r--r-- | mitmproxy/addons/block.py | 2 | ||||
| -rw-r--r-- | mitmproxy/addons/core.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/base.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/image/image_parser.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/encoding.py | 24 | ||||
| -rw-r--r-- | mitmproxy/net/http/message.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/request.py | 4 | ||||
| -rw-r--r-- | mitmproxy/tools/cmdline.py | 1 | ||||
| -rw-r--r-- | mitmproxy/tools/console/commandexecutor.py | 2 | ||||
| -rw-r--r-- | mitmproxy/tools/console/common.py | 359 | ||||
| -rw-r--r-- | mitmproxy/tools/console/consoleaddons.py | 13 | ||||
| -rw-r--r-- | mitmproxy/tools/console/flowlist.py | 10 | ||||
| -rw-r--r-- | mitmproxy/tools/console/flowview.py | 3 | ||||
| -rw-r--r-- | mitmproxy/tools/console/palettes.py | 129 | ||||
| -rw-r--r-- | mitmproxy/tools/web/app.py | 2 | ||||
| -rw-r--r-- | setup.py | 4 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_view.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/net/http/test_encoding.py | 1 | ||||
| -rw-r--r-- | test/mitmproxy/script/test_concurrent.py | 26 |
22 files changed, 549 insertions, 50 deletions
diff --git a/.travis.yml b/.travis.yml index 20afc279..44d452c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,6 @@ matrix: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - pip install tox virtualenv setuptools - - pyenv global system 3.6 script: - tox after_success: diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 88482047..4e2ae47a 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -24,6 +24,9 @@ something like this: Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. +Note: If you are using an iOS device, you should be using the Safari browser +so that it opens the proper prompts for installing the certificate. + ## Installing the mitmproxy CA certificate manually Sometimes using the quick install app is not an option - Java or the iOS diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 33a2f79f..414b4f61 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,10 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + timings = { + k: int(1000 * v) if v != -1 else -1 + for k, v in timings_raw.items() + } # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py index 91f9f709..4ccde0e1 100644 --- a/mitmproxy/addons/block.py +++ b/mitmproxy/addons/block.py @@ -36,4 +36,4 @@ class Block: layer.reply.kill() if ctx.options.block_global and address.is_global: ctx.log.warn("Client connection from %s killed by block_global" % astr) - layer.reply.kill()
\ No newline at end of file + layer.reply.kill() diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index a908dbb3..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 6072dfb7..9b34f3d4 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -37,7 +37,7 @@ class View: def format_pairs( items: typing.Iterable[typing.Tuple[TTextType, TTextType]] -)-> typing.Iterator[TViewLine]: +) -> typing.Iterator[TViewLine]: """ Helper function that accepts a list of (k,v) pairs into a list of diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index fcc50cb5..d5bb404f 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata: entries = block.body.body.entries for entry in entries: comment = entry.bytes - if comment is not b'': + if comment != b'': parts.append(('comment', str(comment))) return parts diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 8cb96e5c..16d399ca 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -9,6 +9,7 @@ from io import BytesIO import gzip import zlib import brotli +import zstandard as zstd from typing import Union, Optional, AnyStr # noqa @@ -52,7 +53,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes: return brotli.compress(content) +def decode_zstd(content: bytes) -> bytes: + if not content: + return b"" + zstd_ctx = zstd.ZstdDecompressor() + try: + return zstd_ctx.decompress(content) + except zstd.ZstdError: + # If the zstd stream is streamed without a size header, + # try decoding with a 10MiB output buffer + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) + + +def encode_zstd(content: bytes) -> bytes: + zstd_ctx = zstd.ZstdCompressor() + return zstd_ctx.compress(content) + + def decode_deflate(content: bytes) -> bytes: """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -170,6 +188,7 @@ custom_decode = { "gzip": decode_gzip, "deflate": decode_deflate, "br": decode_brotli, + "zstd": decode_zstd, } custom_encode = { "none": identity, @@ -177,6 +196,7 @@ custom_encode = { "gzip": encode_gzip, "deflate": encode_deflate, "br": encode_brotli, + "zstd": encode_zstd, } __all__ = ["encode", "decode"] diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 86782e8a..6830c6cd 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -236,7 +236,7 @@ class Message(serializable.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..ef33ca49 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +import time from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union from mitmproxy.coretypes import multidict @@ -101,6 +102,7 @@ class Request(message.Message): ) req.url = url + req.timestamp_start = time.time() # Headers can be list or dict, we differentiate here. if isinstance(headers, dict): @@ -421,7 +423,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate", "br"} + for e in {"gzip", "identity", "deflate", "br", "zstd"} if e in accept_encoding ) ) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 21369a1f..eb4a984d 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,5 +1,6 @@ import argparse + def common_options(parser, opts): parser.add_argument( '--version', diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index 3db03d3e..c738e349 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -34,4 +34,4 @@ class CommandExecutor: ret, ), valign="top" - )
\ No newline at end of file + ) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 5d7ee09d..43ab50cb 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,6 +1,10 @@ import platform import typing +import datetime +import time +import math from functools import lru_cache +from publicsuffix2 import get_sld, get_tld import urwid import urwid.util @@ -97,16 +101,180 @@ if urwid.util.detected_encoding and not IS_WSL: SYMBOL_MARK = u"\u25cf" SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" + SYMBOL_ELLIPSIS = u"\u2026" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" SYMBOL_MARK = "[m]" SYMBOL_UP = "^" SYMBOL_DOWN = " " + SYMBOL_ELLIPSIS = "~" + + +def fixlen(s, maxlen): + if len(s) <= maxlen: + return s.ljust(maxlen) + else: + return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS + + +def fixlen_r(s, maxlen): + if len(s) <= maxlen: + return s.rjust(maxlen) + else: + return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):] + + +class TruncatedText(urwid.Widget): + def __init__(self, text, attr, align='left'): + self.text = text + self.attr = attr + self.align = align + super(TruncatedText, self).__init__() + + def pack(self, size, focus=False): + return (len(self.text), 1) + + def rows(self, size, focus=False): + return 1 + + def render(self, size, focus=False): + text = self.text + attr = self.attr + if self.align == 'right': + text = text[::-1] + attr = attr[::-1] + + text_len = len(text) # TODO: unicode? + if size is not None and len(size) > 0: + width = size[0] + else: + width = text_len + + if width >= text_len: + remaining = width - text_len + if remaining > 0: + c_text = text + ' ' * remaining + c_attr = attr + [('text', remaining)] + else: + c_text = text + c_attr = attr + else: + visible_len = width - len(SYMBOL_ELLIPSIS) + visible_text = text[0:visible_len] + c_text = visible_text + SYMBOL_ELLIPSIS + c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) + + [('focus', len(SYMBOL_ELLIPSIS.encode()))]) + + if self.align == 'right': + c_text = c_text[::-1] + c_attr = c_attr[::-1] + + return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width) + + +def truncated_plain(text, attr, align='left'): + return TruncatedText(text, [(attr, len(text.encode()))], align) + + +# Work around https://github.com/urwid/urwid/pull/330 +def rle_append_beginning_modify(rle, a_r): + """ + Append (a, r) (unpacked from *a_r*) to BEGINNING of rle. + Merge with first run when possible + + MODIFIES rle parameter contents. Returns None. + """ + a, r = a_r + if not rle: + rle[:] = [(a, r)] + else: + al, run = rle[0] + if a == al: + rle[0] = (a, run + r) + else: + rle[0:0] = [(a, r)] + + +def colorize_host(host): + tld = get_tld(host) + sld = get_sld(host) + + attr = [] + + tld_size = len(tld) + sld_size = len(sld) - tld_size + + for letter in reversed(range(len(host))): + character = host[letter] + if tld_size > 0: + style = 'url_domain' + tld_size -= 1 + elif tld_size == 0: + style = 'text' + tld_size -= 1 + elif sld_size > 0: + sld_size -= 1 + style = 'url_extension' + else: + style = 'text' + rle_append_beginning_modify(attr, (style, len(character.encode()))) + return attr + + +def colorize_req(s): + path = s.split('?', 2)[0] + i_query = len(path) + i_last_slash = path.rfind('/') + i_ext = path[i_last_slash + 1:].rfind('.') + i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) + in_val = False + attr = [] + for i in range(len(s)): + c = s[i] + if ((i < i_query and c == '/') or + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): + a = 'url_punctuation' + elif i > i_query: + if in_val: + if c == '&': + in_val = False + a = 'url_punctuation' + else: + a = 'url_query_value' + else: + if c == '=': + in_val = True + a = 'url_punctuation' + else: + a = 'url_query_key' + elif i > i_ext: + a = 'url_extension' + elif i > i_last_slash: + a = 'url_filename' + else: + a = 'text' + urwid.util.rle_append_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_url(url): + parts = url.split('/', 3) + if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':': + return [('error', len(url))] # bad URL + schemes = { + 'http:': 'scheme_http', + 'https:': 'scheme_https', + } + return [ + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) @lru_cache(maxsize=800) -def raw_format_flow(f): +def raw_format_list(f): f = dict(f) pile = [] req = [] @@ -139,8 +307,8 @@ def raw_format_flow(f): url = f["req_url"] - if f["max_url_len"] and len(url) > f["max_url_len"]: - url = url[:f["max_url_len"]] + "…" + if f["cols"] and len(url) > f["cols"]: + url = url[:f["cols"]] + "…" if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): url += " " + f["req_http_version"] @@ -177,7 +345,8 @@ def raw_format_flow(f): if f["resp_ctype"]: resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["roundtrip"], rc)) + pretty_duration = human.pretty_duration(f["duration"]) + resp.append(fcol(pretty_duration, rc)) elif f["err_msg"]: resp.append(fcol(SYMBOL_RETURN, "error")) @@ -193,49 +362,203 @@ def raw_format_flow(f): return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): +@lru_cache(maxsize=800) +def raw_format_table(f): + f = dict(f) + pile = [] + req = [] + + cursor = [' ', 'focus'] + if f.get('resp_is_replay', False): + cursor[0] = SYMBOL_REPLAY + cursor[1] = 'replay' + if f['marked']: + if cursor[0] == ' ': + cursor[0] = SYMBOL_MARK + cursor[1] = 'mark' + if f['focus']: + cursor[0] = '>' + + req.append(fcol(*cursor)) + + if f["two_line"]: + req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) + pile.append(urwid.Columns(req, dividechars=1)) + + req = [] + req.append(fcol(' ', 'text')) + + if f["intercepted"] and not f["acked"]: + uc = "intercept" + elif "resp_code" in f or f["err_msg"] is not None: + uc = "highlight" + else: + uc = "title" + + if f["extended"]: + s = human.format_timestamp(f["req_timestamp"]) + else: + s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") + req.append(fcol(s, uc)) + + methods = { + 'GET': 'method_get', + 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' + } + uc = methods.get(f["req_method"], "method_other") + if f['extended']: + req.append(fcol(f["req_method"], uc)) + if f["req_promise"]: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + else: + if f["req_promise"]: + uc = 'method_http2_push' + req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) + + if f["two_line"]: + req.append(fcol(f["req_http_version"], 'text')) + else: + schemes = { + 'http': 'scheme_http', + 'https': 'scheme_https', + } + req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) + + req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) + req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) + + ret = (' ' * len(SYMBOL_RETURN), 'text') + status = ('', 'text') + content = ('', 'text') + size = ('', 'text') + duration = ('', 'text') + + if "resp_code" in f: + codes = { + 2: "code_200", + 3: "code_300", + 4: "code_400", + 5: "code_500", + } + ccol = codes.get(f["resp_code"] // 100, "code_other") + ret = (SYMBOL_RETURN, ccol) + status = (str(f["resp_code"]), ccol) + + if f["resp_len"] < 0: + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "content_none" + + if f["resp_len"] == -1: + contentdesc = "[content missing]" + else: + contentdesc = "[no content]" + content = (contentdesc, rc) + else: + if f["resp_ctype"]: + ctype = f["resp_ctype"].split(";")[0] + if ctype.endswith('/javascript'): + rc = 'content_script' + elif ctype.startswith('text/'): + rc = 'content_text' + elif (ctype.startswith('image/') or + ctype.startswith('video/') or + ctype.startswith('font/') or + "/x-font-" in ctype): + rc = 'content_media' + elif ctype.endswith('/json') or ctype.endswith('/xml'): + rc = 'content_data' + elif ctype.startswith('application/'): + rc = 'content_raw' + else: + rc = 'content_other' + content = (ctype, rc) + + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) + + size_str = human.pretty_size(f["resp_len"]) + if not f['extended']: + # shorten to 5 chars max + if len(size_str) > 5: + size_str = size_str[0:4].rstrip('.') + size_str[-1:] + size = (size_str, rc) + + if f['duration'] is not None: + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) + duration = (human.pretty_duration(f['duration']), rc) + + elif f["err_msg"]: + status = ('Err', 'error') + content = f["err_msg"], 'error' + + if f["two_line"]: + req.append(fcol(*ret)) + req.append(fcol(fixlen(status[0], 3), status[1])) + req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) + if f['extended']: + req.append(fcol(*size)) + else: + req.append(fcol(fixlen_r(size[0], 5), size[1])) + req.append(fcol(fixlen_r(duration[0], 5), duration[1])) + + pile.append(urwid.Columns(req, dividechars=1, min_width=15)) + + return urwid.Pile(pile) + + +def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'): acked = False if f.reply and f.reply.state == "committed": acked = True - pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in f.metadata else '' d = dict( focus=focus, extended=extended, - max_url_len=max_url_len, + two_line=extended or cols < 100, + cols=cols, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, req_is_replay=f.request.is_replay, - req_method=f.request.method + pushed, + req_method=f.request.method, + req_promise='h2-pushed-stream' in f.metadata, req_url=f.request.pretty_url if hostheader else f.request.url, + req_scheme=f.request.scheme, + req_host=f.request.pretty_host if hostheader else f.request.host, + req_path=f.request.path, req_http_version=f.request.http_version, err_msg=f.error.msg if f.error else None, marked=f.marked, ) if f.response: if f.response.raw_content: + content_len = len(f.response.raw_content) contentdesc = human.pretty_size(len(f.response.raw_content)) elif f.response.raw_content is None: + content_len = -1 contentdesc = "[content missing]" else: + content_len = -2 contentdesc = "[no content]" - duration = 0 + + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start - roundtrip = human.pretty_duration(duration) d.update(dict( resp_code=f.response.status_code, resp_reason=f.response.reason, resp_is_replay=f.response.is_replay, + resp_len=content_len, + resp_ctype=f.response.headers.get("content-type"), resp_clen=contentdesc, - roundtrip=roundtrip, + duration=duration, )) - t = f.response.headers.get("content-type") - if t: - d["resp_ctype"] = t.split(";")[0] - else: - d["resp_ctype"] = "" - - return raw_format_flow(tuple(sorted(d.items()))) + if ((layout == 'default' and cols < 100) or layout == "list"): + return raw_format_list(tuple(sorted(d.items()))) + else: + return raw_format_table(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..13f3ff7d 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -37,6 +37,12 @@ console_layouts = [ "horizontal", ] +console_flowlist_layout = [ + "default", + "table", + "list" +] + class UnsupportedLog: """ @@ -114,6 +120,13 @@ class ConsoleAddon: "Console mouse interaction." ) + loader.add_option( + "console_flowlist_layout", + str, "default", + "Set the flowlist layout", + choices=sorted(console_flowlist_layout) + ) + @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: """ diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e947a582..9650c0d3 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -18,7 +18,8 @@ class FlowItem(urwid.WidgetWrap): self.flow, self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) def selectable(self): @@ -84,6 +85,10 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): ) -> None: self.master: "mitmproxy.tools.console.master.ConsoleMaster" = master super().__init__(FlowListWalker(master)) + self.master.options.subscribe( + self.set_flowlist_layout, + ["console_flowlist_layout"] + ) def keypress(self, size, key): if key == "m_start": @@ -96,3 +101,6 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): def view_changed(self): self.body.view_changed() + + def set_flowlist_layout(self, opts, updated): + self.master.ui.clear() diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index b4e3876f..807c9714 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -38,7 +38,8 @@ class FlowViewHeader(urwid.WidgetWrap): False, extended=True, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 7930c4a3..4eee7692 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,7 +22,12 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'focus', + 'method', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', + 'scheme_http', 'scheme_https', 'scheme_other', + 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', + 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', + 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', @@ -36,6 +41,7 @@ class Palette: # Commander 'commander_command', 'commander_invalid', 'commander_hint' ] + _fields.extend(['gradient_%02d' % i for i in range(100)]) high: typing.Mapping[str, typing.Sequence[str]] = None def palette(self, transparent): @@ -68,6 +74,27 @@ class Palette: return l +def gen_gradient(palette, cols): + for i in range(100): + palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default') + + +def gen_rgb_gradient(palette, cols): + parts = len(cols) - 1 + for i in range(100): + p = i / 100 + idx = int(p * parts) + t0 = cols[idx] + t1 = cols[idx + 1] + pp = p * parts % 1 + t = ( + round(t0[0] + (t1[0] - t0[0]) * pp), + round(t0[1] + (t1[1] - t0[1]) * pp), + round(t0[2] + (t1[2] - t0[2]) * pp), + ) + palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default') + + class LowDark(Palette): """ @@ -95,6 +122,33 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), + method_other = ('dark magenta', 'default'), + method_http2_push = ('dark gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('dark green', 'default'), + scheme_other = ('dark magenta', 'default'), + + url_punctuation = ('light gray', 'default'), + url_domain = ('white', 'default'), + url_filename = ('dark cyan', 'default'), + url_extension = ('light gray', 'default'), + url_query_key = ('white', 'default'), + url_query_value = ('light gray', 'default'), + + content_none = ('dark gray', 'default'), + content_text = ('light gray', 'default'), + content_script = ('dark green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('dark red', 'default'), + content_other = ('dark magenta', 'default'), + focus = ('yellow', 'default'), code_200 = ('dark green', 'default'), @@ -127,6 +181,7 @@ class LowDark(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('dark gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Dark(LowDark): @@ -167,6 +222,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -198,6 +280,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -256,7 +339,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -311,9 +414,28 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections - method = (sol_cyan, 'default'), focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = (sol_base01, 'default'), + + url_punctuation = ('h242', 'default'), + url_domain = ('h252', 'default'), + url_filename = ('h132', 'default'), + url_extension = ('h96', 'default'), + url_query_key = ('h37', 'default'), + url_query_value = ('h30', 'default'), + + content_none = (sol_base01, 'default'), + content_text = (sol_base1, 'default'), + content_media = (sol_blue, 'default'), + code_200 = (sol_green, 'default'), code_300 = (sol_blue, 'default'), code_400 = (sol_orange, 'default',), @@ -342,6 +464,7 @@ class SolarizedDark(LowDark): commander_invalid = (sol_orange, 'default'), commander_hint = (sol_base00, 'default'), ) + gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)]) DEFAULT = "dark" diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 6e6b6223..6bfce34e 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -432,7 +432,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "upstream_cert", + "intercept", "showhost", "upstream_cert", "ssl_insecure", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } @@ -81,6 +81,8 @@ setup( "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.13.0,<0.14.0", + "publicsuffix2~=2.20", + "zstandard>=0.11.0,<0.13.0", ], extras_require={ ':sys_platform == "win32"': [ @@ -88,7 +90,7 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5,<3.7", + "flake8>=3.5,<=3.7.8", "Flask>=1.0,<1.1", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 976c14b7..f5088a68 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -471,7 +471,7 @@ def test_focus(): v = view.View() v.add([tft()]) f = view.Focus(v) - assert f.index is 0 + assert f.index == 0 assert f.flow is v[0] # Start empty diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index 8dac12cb..7f768f39 100644 --- a/test/mitmproxy/net/http/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -19,6 +19,7 @@ def test_identity(encoder): 'gzip', 'br', 'deflate', + 'zstd', ]) def test_encoders(encoder): """ diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 3ec58760..70d41511 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest): assert await tctx.master.await_log("decorator not supported") def test_concurrent_class(self, tdata): - with taddons.context() as tctx: - sc = tctx.script( - tdata.path( - "mitmproxy/data/addonscripts/concurrent_decorator_class.py" - ) + with taddons.context() as tctx: + sc = tctx.script( + tdata.path( + "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) - f1, f2 = tflow.tflow(), tflow.tflow() - tctx.cycle(sc, f1) - tctx.cycle(sc, f2) - start = time.time() - while time.time() - start < 5: - if f1.reply.state == f2.reply.state == "committed": - return - raise ValueError("Script never acked") + ) + f1, f2 = tflow.tflow(), tflow.tflow() + tctx.cycle(sc, f1) + tctx.cycle(sc, f2) + start = time.time() + while time.time() - start < 5: + if f1.reply.state == f2.reply.state == "committed": + return + raise ValueError("Script never acked") |