integrate simplified contentviews

6 files changed, 137 insertions, 132 deletions

diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py
index b1367e12..a98c9b46 100644
--- a/mitmproxy/builtins/dumper.py
+++ b/mitmproxy/builtins/dumper.py
@@ -63,20 +63,12 @@ class Dumper(object):
             )
             self.echo(headers, ident=4)
         if self.flow_detail >= 3:
-            try:
-                content = message.content
-            except ValueError:
-                content = message.get_content(strict=False)
-
-            if content is None:
-                self.echo("(content missing)", ident=4)
-            elif content:
-                self.echo("")
-
-                _, lines = contentviews.get_content_view_with_message_encoding(
-                    message,
-                    contentviews.get("Auto")
+                _, lines, error = contentviews.get_message_content_view(
+                    contentviews.get("Auto"),
+                    message
                 )
+                if error:
+                    ctx.log.debug(error)
 
                 styles = dict(
                     highlight=dict(bold=True),
@@ -95,13 +87,13 @@ class Dumper(object):
                 else:
                     lines_to_echo = lines
 
-                lines_to_echo = list(lines_to_echo)
-
                 content = u"\r\n".join(
                     u"".join(colorful(line)) for line in lines_to_echo
                 )
+                if content:
+                    self.echo("")
+                    self.echo(content)
 
-                self.echo(content)
                 if next(lines, None):
                     self.echo("(cut off)", ident=4, dim=True)
 
@@ -251,4 +243,4 @@ class Dumper(object):
             server=repr(f.server_conn.address),
             direction=direction,
         ))
-        self._echo_message(message)
+        self._echo_message(message)
\ No newline at end of file
diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py
index c1001a5a..323aefd2 100644
--- a/mitmproxy/console/flowview.py
+++ b/mitmproxy/console/flowview.py
@@ -80,7 +80,7 @@ def _mkhelp():
         ("r", "replay request"),
         ("V", "revert changes to request"),
         ("v", "view body in external viewer"),
-        ("w", "save all flows matching current limit"),
+        ("w", "save all flows matching current view filter"),
         ("W", "save this flow"),
         ("x", "delete body"),
         ("z", "encode/decode a request/response"),
@@ -206,10 +206,11 @@ class FlowView(tabs.Tabs):
             )
 
     def _get_content_view(self, message, viewmode, max_lines, _):
-
-        description, lines = contentviews.get_content_view_with_message_encoding(
-            message, viewmode
+        description, lines, error = contentviews.get_message_content_view(
+            viewmode, message
         )
+        if error:
+            signals.add_log(error, "error")
         # Give hint that you have to tab for the response.
         if description == "No content" and isinstance(message, models.HTTPRequest):
             description = "No request content (press tab to view response)"
@@ -687,6 +688,7 @@ class FlowView(tabs.Tabs):
                     keys = (
                         ("gzip", "z"),
                         ("deflate", "d"),
+                        ("brotli", "b"),
                     ),
                     callback = self.encode_callback,
                     args = (conn,)
@@ -700,6 +702,7 @@ class FlowView(tabs.Tabs):
         encoding_map = {
             "z": "gzip",
             "d": "deflate",
+            "b": "brotli",
         }
         conn.encode(encoding_map[key])
-        signals.flow_change.send(self, flow = self.flow)
+        signals.flow_change.send(self, flow = self.flow)
\ No newline at end of file
diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py
index 88d979e4..807638dc 100644
--- a/mitmproxy/contentviews.py
+++ b/mitmproxy/contentviews.py
@@ -14,31 +14,27 @@ requests, the query parameters are passed as the ``query`` keyword argument.
 """
 from __future__ import absolute_import, print_function, division
 
+import cssutils
 import datetime
+import html2text
+import jsbeautifier
 import json
 import logging
-import subprocess
-import sys
-import math
-
-from typing import Mapping  # noqa
-
-import html2text
 import lxml.etree
 import lxml.html
 import six
+import subprocess
+import traceback
 from PIL import ExifTags
 from PIL import Image
-from six import BytesIO
-
-from mitmproxy import models
 from mitmproxy import exceptions
-from mitmproxy.contrib import jsbeautifier
 from mitmproxy.contrib.wbxml import ASCommandResponse
 from netlib import http
 from netlib import multidict
-from netlib.http import url
 from netlib import strutils
+from netlib.http import url
+from six import BytesIO
+from typing import Mapping  # noqa
 
 try:
     import pyamf
@@ -46,17 +42,6 @@ try:
 except ImportError:  # pragma no cover
     pyamf = None
 
-try:
-    import cssutils
-except ImportError:  # pragma no cover
-    cssutils = None
-else:
-    cssutils.log.setLevel(logging.CRITICAL)
-
-    cssutils.ser.prefs.keepComments = True
-    cssutils.ser.prefs.omitLastSemicolon = False
-    cssutils.ser.prefs.indentClosingBrace = False
-    cssutils.ser.prefs.validOnly = False
 
 # Default view cutoff *in lines*
 VIEW_CUTOFF = 512
@@ -273,7 +258,7 @@ class ViewHTMLOutline(View):
     content_types = ["text/html"]
 
     def __call__(self, data, **metadata):
-        data = data.decode("utf-8")
+        data = data.decode("utf-8", "replace")
         h = html2text.HTML2Text(baseurl="")
         h.ignore_images = True
         h.body_width = 0
@@ -400,6 +385,7 @@ class ViewJavaScript(View):
     def __call__(self, data, **metadata):
         opts = jsbeautifier.default_options()
         opts.indent_size = 2
+        data = data.decode("utf-8", "replace")
         res = jsbeautifier.beautify(data, opts)
         return "JavaScript", format_text(res)
 
@@ -412,11 +398,14 @@ class ViewCSS(View):
     ]
 
     def __call__(self, data, **metadata):
-        if cssutils:
-            sheet = cssutils.parseString(data)
-            beautified = sheet.cssText
-        else:
-            beautified = data
+        cssutils.log.setLevel(logging.CRITICAL)
+        cssutils.ser.prefs.keepComments = True
+        cssutils.ser.prefs.omitLastSemicolon = False
+        cssutils.ser.prefs.indentClosingBrace = False
+        cssutils.ser.prefs.validOnly = False
+
+        sheet = cssutils.parseString(data)
+        beautified = sheet.cssText
 
         return "CSS", format_text(beautified)
 
@@ -619,35 +608,38 @@ def safe_to_print(lines, encoding="utf8"):
         yield clean_line
 
 
-def get_content_view_with_message_encoding(message, viewmode):
+def get_message_content_view(viewmode, message):
+    """
+    Like get_content_view, but also handles message encoding.
+    """
     try:
         content = message.content
-        if content != message.raw_content:
+    except ValueError:
+        content = message.raw_content
+        enc = "[cannot decode]"
+    else:
+        if isinstance(message, http.Message) and content != message.raw_content:
             enc = "[decoded {}]".format(
                 message.headers.get("content-encoding")
             )
         else:
             enc = None
-    except ValueError:
-        content = message.raw_content
-        enc = "[cannot decode]"
-    try:
-        query = None
-        if isinstance(message, models.HTTPRequest):
-            query = message.query
-        description, lines = get_content_view(
-            viewmode, content, headers=message.headers, query=query
-        )
-    except exceptions.ContentViewException:
-        description, lines = get_content_view(
-            get("Raw"), content, headers=message.headers
-        )
-        description = description.replace("Raw", "Couldn't parse: falling back to Raw")
+
+    if content is None:
+        return "", iter([[("error", "content missing")]]), None
+
+    query = message.query if isinstance(message, http.Request) else None
+    headers = message.headers if isinstance(message, http.Message) else None
+
+    description, lines, error = get_content_view(
+        viewmode, content, headers=headers, query=query
+    )
 
     if enc:
-        description = " ".join([enc, description])
+        description = "{} {}".format(enc, description)
+
+    return description, lines, error
 
-    return description, lines
 
 def get_content_view(viewmode, data, **metadata):
     """
@@ -656,24 +648,24 @@ def get_content_view(viewmode, data, **metadata):
             data, **metadata: arguments passed to View instance.
 
         Returns:
-            A (description, content generator) tuple.
+            A (description, content generator, error) tuple.
+            If the content view raised an exception generating the view,
+            the exception is returned in error and the flow is formatted in raw mode.
             In contrast to calling the views directly, text is always safe-to-print unicode.
-
-        Raises:
-            ContentViewException, if the content view threw an error.
     """
     try:
         ret = viewmode(data, **metadata)
+        if ret is None:
+            ret = "Couldn't parse: falling back to Raw", get("Raw")(data, **metadata)[1]
+        desc, content = ret
+        error = None
     # Third-party viewers can fail in unexpected ways...
-    except Exception as e:
-        six.reraise(
-            exceptions.ContentViewException,
-            exceptions.ContentViewException(str(e)),
-            sys.exc_info()[2]
-        )
-    if not ret:
+    except Exception:
         desc = "Couldn't parse: falling back to Raw"
         _, content = get("Raw")(data, **metadata)
-    else:
-        desc, content = ret
-    return desc, safe_to_print(content)
+        error = "{} Content viewer failed: \n{}".format(
+            getattr(viewmode, "name"),
+            traceback.format_exc()
+        )
+
+    return desc, safe_to_print(content), error
\ No newline at end of file
diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py
index af95f4d0..fdf4ef8f 100644
--- a/netlib/http/__init__.py
+++ b/netlib/http/__init__.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, print_function, division
 from netlib.http.request import Request
 from netlib.http.response import Response
+from netlib.http.message import Message
 from netlib.http.headers import Headers, parse_content_type
 from netlib.http.message import decoded
 from netlib.http import http1, http2, status_codes, multipart
@@ -11,4 +12,4 @@ __all__ = [
     "Headers", "parse_content_type",
     "decoded",
     "http1", "http2", "status_codes", "multipart",
-]
+]
\ No newline at end of file
diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py
index 6287fe86..2d551bab 100644
--- a/test/mitmproxy/builtins/test_dumper.py
+++ b/test/mitmproxy/builtins/test_dumper.py
@@ -15,7 +15,7 @@ class TestDumper(mastertest.MasterTest):
         d = dumper.Dumper()
         sio = StringIO()
 
-        updated = set(["tfile", "flow_detail"])
+        updated = {"tfile", "flow_detail"}
         d.configure(dump.Options(tfile = sio, flow_detail = 0), updated)
         d.response(tutils.tflow())
         assert not sio.getvalue()
@@ -66,10 +66,9 @@ class TestDumper(mastertest.MasterTest):
 
 
 class TestContentView(mastertest.MasterTest):
-    @mock.patch("mitmproxy.contentviews.get_content_view")
-    def test_contentview(self, get_content_view):
-        se = exceptions.ContentViewException(""), ("x", iter([]))
-        get_content_view.side_effect = se
+    @mock.patch("mitmproxy.contentviews.ViewAuto.__call__")
+    def test_contentview(self, view_auto):
+        view_auto.side_effect = exceptions.ContentViewException("")
 
         s = state.State()
         sio = StringIO()
@@ -82,4 +81,4 @@ class TestContentView(mastertest.MasterTest):
         d = dumper.Dumper()
         m.addons.add(o, d)
         self.invoke(m, "response", tutils.tflow())
-        assert "Content viewer failed" in m.event_log[0][1]
+        assert "Content viewer failed" in m.event_log[0][1]
\ No newline at end of file
diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py
index aad53b37..8e2042fb 100644
--- a/test/mitmproxy/test_contentview.py
+++ b/test/mitmproxy/test_contentview.py
@@ -1,3 +1,4 @@
+import mock
 from mitmproxy.exceptions import ContentViewException
 from netlib.http import Headers
 from netlib.http import url
@@ -5,6 +6,7 @@ from netlib import multidict
 
 import mitmproxy.contentviews as cv
 from . import tutils
+import netlib.tutils
 
 try:
     import pyamf
@@ -78,6 +80,7 @@ class TestContentView:
         v = cv.ViewHTMLOutline()
         s = b"<html><br><br></br><p>one</p></html>"
         assert v(s)
+        assert v(b'\xfe')
 
     def test_view_json(self):
         cv.VIEW_CUTOFF = 100
@@ -106,9 +109,10 @@ class TestContentView:
 
     def test_view_javascript(self):
         v = cv.ViewJavaScript()
-        assert v("[1, 2, 3]")
-        assert v("[1, 2, 3")
-        assert v("function(a){[1, 2, 3]}")
+        assert v(b"[1, 2, 3]")
+        assert v(b"[1, 2, 3")
+        assert v(b"function(a){[1, 2, 3]}")
+        assert v(b"\xfe")  # invalid utf-8
 
     def test_view_css(self):
         v = cv.ViewCSS()
@@ -178,43 +182,6 @@ Larry
         assert f[0] == "Query"
         assert [x for x in f[1]] == [[("header", "foo: "), ("text", "bar")]]
 
-    def test_get_content_view(self):
-        r = cv.get_content_view(
-            cv.get("Raw"),
-            b"[1, 2, 3]",
-            headers=Headers(content_type="application/json")
-        )
-        assert "Raw" in r[0]
-
-        r = cv.get_content_view(
-            cv.get("Auto"),
-            b"[1, 2, 3]",
-            headers=Headers(content_type="application/json")
-        )
-        assert r[0] == "JSON"
-
-        r = cv.get_content_view(
-            cv.get("Auto"),
-            b"[1, 2",
-            headers=Headers(content_type="application/json")
-        )
-        assert "Raw" in r[0]
-
-        r = cv.get_content_view(
-            cv.get("Auto"),
-            b"[1, 2, 3]",
-            headers=Headers(content_type="application/vnd.api+json")
-        )
-        assert r[0] == "JSON"
-
-        tutils.raises(
-            ContentViewException,
-            cv.get_content_view,
-            cv.get("AMF"),
-            b"[1, 2",
-            headers=Headers()
-        )
-
     def test_add_cv(self):
         class TestContentView(cv.View):
             name = "test"
@@ -231,6 +198,57 @@ Larry
         )
 
 
+def test_get_content_view():
+    desc, lines, err = cv.get_content_view(
+        cv.get("Raw"),
+        b"[1, 2, 3]",
+    )
+    assert "Raw" in desc
+    assert list(lines)
+    assert not err
+
+    desc, lines, err = cv.get_content_view(
+        cv.get("Auto"),
+        b"[1, 2, 3]",
+        headers=Headers(content_type="application/json")
+    )
+    assert desc == "JSON"
+
+    desc, lines, err = cv.get_content_view(
+        cv.get("JSON"),
+        b"[1, 2",
+    )
+    assert "Couldn't parse" in desc
+
+    with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto:
+        view_auto.side_effect = ValueError
+
+        desc, lines, err = cv.get_content_view(
+            cv.get("JSON"),
+            b"[1, 2",
+        )
+        assert err
+        assert "Couldn't parse" in desc
+
+
+def test_get_message_content_view():
+    r = netlib.tutils.treq()
+    desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
+    assert desc == "Raw"
+
+    r.encode("gzip")
+    desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
+    assert desc == "[decoded gzip] Raw"
+
+    r.headers["content-encoding"] = "deflate"
+    desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
+    assert desc == "[cannot decode] Raw"
+
+    r.content = None
+    desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
+    assert list(lines) == [[("error", "content missing")]]
+
+
 if pyamf:
     def test_view_amf_request():
         v = cv.ViewAMF()
@@ -264,4 +282,4 @@ def test_pretty_json():
     assert cv.pretty_json(b'{"foo": 1}')
     assert not cv.pretty_json(b"moo")
     assert cv.pretty_json(b'{"foo" : "\xe4\xb8\x96\xe7\x95\x8c"}')  # utf8 with chinese characters
-    assert not cv.pretty_json(b'{"foo" : "\xFF"}')
+    assert not cv.pretty_json(b'{"foo" : "\xFF"}')
\ No newline at end of file