diff options
| -rw-r--r-- | mitmproxy/builtins/dumper.py | 26 | ||||
| -rw-r--r-- | mitmproxy/console/flowview.py | 13 | ||||
| -rw-r--r-- | mitmproxy/contentviews.py | 116 | ||||
| -rw-r--r-- | netlib/http/__init__.py | 3 | ||||
| -rw-r--r-- | test/mitmproxy/builtins/test_dumper.py | 11 | ||||
| -rw-r--r-- | test/mitmproxy/test_contentview.py | 100 |
6 files changed, 137 insertions, 132 deletions
diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py index b1367e12..a98c9b46 100644 --- a/mitmproxy/builtins/dumper.py +++ b/mitmproxy/builtins/dumper.py @@ -63,20 +63,12 @@ class Dumper(object): ) self.echo(headers, ident=4) if self.flow_detail >= 3: - try: - content = message.content - except ValueError: - content = message.get_content(strict=False) - - if content is None: - self.echo("(content missing)", ident=4) - elif content: - self.echo("") - - _, lines = contentviews.get_content_view_with_message_encoding( - message, - contentviews.get("Auto") + _, lines, error = contentviews.get_message_content_view( + contentviews.get("Auto"), + message ) + if error: + ctx.log.debug(error) styles = dict( highlight=dict(bold=True), @@ -95,13 +87,13 @@ class Dumper(object): else: lines_to_echo = lines - lines_to_echo = list(lines_to_echo) - content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) + if content: + self.echo("") + self.echo(content) - self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) @@ -251,4 +243,4 @@ class Dumper(object): server=repr(f.server_conn.address), direction=direction, )) - self._echo_message(message) + self._echo_message(message)
\ No newline at end of file diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index c1001a5a..323aefd2 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -80,7 +80,7 @@ def _mkhelp(): ("r", "replay request"), ("V", "revert changes to request"), ("v", "view body in external viewer"), - ("w", "save all flows matching current limit"), + ("w", "save all flows matching current view filter"), ("W", "save this flow"), ("x", "delete body"), ("z", "encode/decode a request/response"), @@ -206,10 +206,11 @@ class FlowView(tabs.Tabs): ) def _get_content_view(self, message, viewmode, max_lines, _): - - description, lines = contentviews.get_content_view_with_message_encoding( - message, viewmode + description, lines, error = contentviews.get_message_content_view( + viewmode, message ) + if error: + signals.add_log(error, "error") # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, models.HTTPRequest): description = "No request content (press tab to view response)" @@ -687,6 +688,7 @@ class FlowView(tabs.Tabs): keys = ( ("gzip", "z"), ("deflate", "d"), + ("brotli", "b"), ), callback = self.encode_callback, args = (conn,) @@ -700,6 +702,7 @@ class FlowView(tabs.Tabs): encoding_map = { "z": "gzip", "d": "deflate", + "b": "brotli", } conn.encode(encoding_map[key]) - signals.flow_change.send(self, flow = self.flow) + signals.flow_change.send(self, flow = self.flow)
\ No newline at end of file diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 88d979e4..807638dc 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -14,31 +14,27 @@ requests, the query parameters are passed as the ``query`` keyword argument. """ from __future__ import absolute_import, print_function, division +import cssutils import datetime +import html2text +import jsbeautifier import json import logging -import subprocess -import sys -import math - -from typing import Mapping # noqa - -import html2text import lxml.etree import lxml.html import six +import subprocess +import traceback from PIL import ExifTags from PIL import Image -from six import BytesIO - -from mitmproxy import models from mitmproxy import exceptions -from mitmproxy.contrib import jsbeautifier from mitmproxy.contrib.wbxml import ASCommandResponse from netlib import http from netlib import multidict -from netlib.http import url from netlib import strutils +from netlib.http import url +from six import BytesIO +from typing import Mapping # noqa try: import pyamf @@ -46,17 +42,6 @@ try: except ImportError: # pragma no cover pyamf = None -try: - import cssutils -except ImportError: # pragma no cover - cssutils = None -else: - cssutils.log.setLevel(logging.CRITICAL) - - cssutils.ser.prefs.keepComments = True - cssutils.ser.prefs.omitLastSemicolon = False - cssutils.ser.prefs.indentClosingBrace = False - cssutils.ser.prefs.validOnly = False # Default view cutoff *in lines* VIEW_CUTOFF = 512 @@ -273,7 +258,7 @@ class ViewHTMLOutline(View): content_types = ["text/html"] def __call__(self, data, **metadata): - data = data.decode("utf-8") + data = data.decode("utf-8", "replace") h = html2text.HTML2Text(baseurl="") h.ignore_images = True h.body_width = 0 @@ -400,6 +385,7 @@ class ViewJavaScript(View): def __call__(self, data, **metadata): opts = jsbeautifier.default_options() opts.indent_size = 2 + data = data.decode("utf-8", "replace") res = jsbeautifier.beautify(data, opts) return "JavaScript", format_text(res) @@ -412,11 +398,14 @@ class ViewCSS(View): ] def __call__(self, data, **metadata): - if cssutils: - sheet = cssutils.parseString(data) - beautified = sheet.cssText - else: - beautified = data + cssutils.log.setLevel(logging.CRITICAL) + cssutils.ser.prefs.keepComments = True + cssutils.ser.prefs.omitLastSemicolon = False + cssutils.ser.prefs.indentClosingBrace = False + cssutils.ser.prefs.validOnly = False + + sheet = cssutils.parseString(data) + beautified = sheet.cssText return "CSS", format_text(beautified) @@ -619,35 +608,38 @@ def safe_to_print(lines, encoding="utf8"): yield clean_line -def get_content_view_with_message_encoding(message, viewmode): +def get_message_content_view(viewmode, message): + """ + Like get_content_view, but also handles message encoding. + """ try: content = message.content - if content != message.raw_content: + except ValueError: + content = message.raw_content + enc = "[cannot decode]" + else: + if isinstance(message, http.Message) and content != message.raw_content: enc = "[decoded {}]".format( message.headers.get("content-encoding") ) else: enc = None - except ValueError: - content = message.raw_content - enc = "[cannot decode]" - try: - query = None - if isinstance(message, models.HTTPRequest): - query = message.query - description, lines = get_content_view( - viewmode, content, headers=message.headers, query=query - ) - except exceptions.ContentViewException: - description, lines = get_content_view( - get("Raw"), content, headers=message.headers - ) - description = description.replace("Raw", "Couldn't parse: falling back to Raw") + + if content is None: + return "", iter([[("error", "content missing")]]), None + + query = message.query if isinstance(message, http.Request) else None + headers = message.headers if isinstance(message, http.Message) else None + + description, lines, error = get_content_view( + viewmode, content, headers=headers, query=query + ) if enc: - description = " ".join([enc, description]) + description = "{} {}".format(enc, description) + + return description, lines, error - return description, lines def get_content_view(viewmode, data, **metadata): """ @@ -656,24 +648,24 @@ def get_content_view(viewmode, data, **metadata): data, **metadata: arguments passed to View instance. Returns: - A (description, content generator) tuple. + A (description, content generator, error) tuple. + If the content view raised an exception generating the view, + the exception is returned in error and the flow is formatted in raw mode. In contrast to calling the views directly, text is always safe-to-print unicode. - - Raises: - ContentViewException, if the content view threw an error. """ try: ret = viewmode(data, **metadata) + if ret is None: + ret = "Couldn't parse: falling back to Raw", get("Raw")(data, **metadata)[1] + desc, content = ret + error = None # Third-party viewers can fail in unexpected ways... - except Exception as e: - six.reraise( - exceptions.ContentViewException, - exceptions.ContentViewException(str(e)), - sys.exc_info()[2] - ) - if not ret: + except Exception: desc = "Couldn't parse: falling back to Raw" _, content = get("Raw")(data, **metadata) - else: - desc, content = ret - return desc, safe_to_print(content) + error = "{} Content viewer failed: \n{}".format( + getattr(viewmode, "name"), + traceback.format_exc() + ) + + return desc, safe_to_print(content), error
\ No newline at end of file diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index af95f4d0..fdf4ef8f 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, print_function, division from netlib.http.request import Request from netlib.http.response import Response +from netlib.http.message import Message from netlib.http.headers import Headers, parse_content_type from netlib.http.message import decoded from netlib.http import http1, http2, status_codes, multipart @@ -11,4 +12,4 @@ __all__ = [ "Headers", "parse_content_type", "decoded", "http1", "http2", "status_codes", "multipart", -] +]
\ No newline at end of file diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py index 6287fe86..2d551bab 100644 --- a/test/mitmproxy/builtins/test_dumper.py +++ b/test/mitmproxy/builtins/test_dumper.py @@ -15,7 +15,7 @@ class TestDumper(mastertest.MasterTest): d = dumper.Dumper() sio = StringIO() - updated = set(["tfile", "flow_detail"]) + updated = {"tfile", "flow_detail"} d.configure(dump.Options(tfile = sio, flow_detail = 0), updated) d.response(tutils.tflow()) assert not sio.getvalue() @@ -66,10 +66,9 @@ class TestDumper(mastertest.MasterTest): class TestContentView(mastertest.MasterTest): - @mock.patch("mitmproxy.contentviews.get_content_view") - def test_contentview(self, get_content_view): - se = exceptions.ContentViewException(""), ("x", iter([])) - get_content_view.side_effect = se + @mock.patch("mitmproxy.contentviews.ViewAuto.__call__") + def test_contentview(self, view_auto): + view_auto.side_effect = exceptions.ContentViewException("") s = state.State() sio = StringIO() @@ -82,4 +81,4 @@ class TestContentView(mastertest.MasterTest): d = dumper.Dumper() m.addons.add(o, d) self.invoke(m, "response", tutils.tflow()) - assert "Content viewer failed" in m.event_log[0][1] + assert "Content viewer failed" in m.event_log[0][1]
\ No newline at end of file diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index aad53b37..8e2042fb 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,3 +1,4 @@ +import mock from mitmproxy.exceptions import ContentViewException from netlib.http import Headers from netlib.http import url @@ -5,6 +6,7 @@ from netlib import multidict import mitmproxy.contentviews as cv from . import tutils +import netlib.tutils try: import pyamf @@ -78,6 +80,7 @@ class TestContentView: v = cv.ViewHTMLOutline() s = b"<html><br><br></br><p>one</p></html>" assert v(s) + assert v(b'\xfe') def test_view_json(self): cv.VIEW_CUTOFF = 100 @@ -106,9 +109,10 @@ class TestContentView: def test_view_javascript(self): v = cv.ViewJavaScript() - assert v("[1, 2, 3]") - assert v("[1, 2, 3") - assert v("function(a){[1, 2, 3]}") + assert v(b"[1, 2, 3]") + assert v(b"[1, 2, 3") + assert v(b"function(a){[1, 2, 3]}") + assert v(b"\xfe") # invalid utf-8 def test_view_css(self): v = cv.ViewCSS() @@ -178,43 +182,6 @@ Larry assert f[0] == "Query" assert [x for x in f[1]] == [[("header", "foo: "), ("text", "bar")]] - def test_get_content_view(self): - r = cv.get_content_view( - cv.get("Raw"), - b"[1, 2, 3]", - headers=Headers(content_type="application/json") - ) - assert "Raw" in r[0] - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2, 3]", - headers=Headers(content_type="application/json") - ) - assert r[0] == "JSON" - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2", - headers=Headers(content_type="application/json") - ) - assert "Raw" in r[0] - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2, 3]", - headers=Headers(content_type="application/vnd.api+json") - ) - assert r[0] == "JSON" - - tutils.raises( - ContentViewException, - cv.get_content_view, - cv.get("AMF"), - b"[1, 2", - headers=Headers() - ) - def test_add_cv(self): class TestContentView(cv.View): name = "test" @@ -231,6 +198,57 @@ Larry ) +def test_get_content_view(): + desc, lines, err = cv.get_content_view( + cv.get("Raw"), + b"[1, 2, 3]", + ) + assert "Raw" in desc + assert list(lines) + assert not err + + desc, lines, err = cv.get_content_view( + cv.get("Auto"), + b"[1, 2, 3]", + headers=Headers(content_type="application/json") + ) + assert desc == "JSON" + + desc, lines, err = cv.get_content_view( + cv.get("JSON"), + b"[1, 2", + ) + assert "Couldn't parse" in desc + + with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto: + view_auto.side_effect = ValueError + + desc, lines, err = cv.get_content_view( + cv.get("JSON"), + b"[1, 2", + ) + assert err + assert "Couldn't parse" in desc + + +def test_get_message_content_view(): + r = netlib.tutils.treq() + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "Raw" + + r.encode("gzip") + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "[decoded gzip] Raw" + + r.headers["content-encoding"] = "deflate" + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "[cannot decode] Raw" + + r.content = None + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert list(lines) == [[("error", "content missing")]] + + if pyamf: def test_view_amf_request(): v = cv.ViewAMF() @@ -264,4 +282,4 @@ def test_pretty_json(): assert cv.pretty_json(b'{"foo": 1}') assert not cv.pretty_json(b"moo") assert cv.pretty_json(b'{"foo" : "\xe4\xb8\x96\xe7\x95\x8c"}') # utf8 with chinese characters - assert not cv.pretty_json(b'{"foo" : "\xFF"}') + assert not cv.pretty_json(b'{"foo" : "\xFF"}')
\ No newline at end of file |