Merge branch 'master' into admin

30 files changed, 310 insertions, 110 deletions

diff --git a/.appveyor.yml b/.appveyor.yml
index b95b23e3..3725ea3f 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -34,7 +34,7 @@ test_script:
 
   - ps: |
       if(
-        ($Env:TOXENV -match "py35") -and
+        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
         (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
       ) {
         tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
diff --git a/docs/index.rst b/docs/index.rst
index cd32a1f6..a40a5f62 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -17,6 +17,7 @@
 
     mitmproxy
     mitmdump
+    mitmweb
     config
 
 .. toctree::
diff --git a/docs/install.rst b/docs/install.rst
index 1fe09aca..93b3c364 100644
--- a/docs/install.rst
+++ b/docs/install.rst
@@ -3,118 +3,144 @@
 Installation
 ============
 
-.. _install-ubuntu:
+Please follow the steps for your operating system.
 
-Installation On Ubuntu
-----------------------
+Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or
+:ref:`mitmweb` from a terminal.
 
-Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Ubuntu 16.04.
+
+.. _install-macos:
+
+Installation on macOS
+---------------------
+
+You can use Homebrew to install everything:
 
 .. code:: bash
 
-   sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev
-   sudo pip3 install mitmproxy  # or pip install --user mitmproxy
+    brew install mitmproxy
 
-On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install a newer version of Python.
-mitmproxy requires Python 3.5 or higher. Please take a look at pyenv_.
-Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
+Or you can download the pre-built binary packages from `mitmproxy.org`_.
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
+.. _install-windows:
 
-.. _install-fedora:
+Installation on Windows
+-----------------------
 
-Installation On Fedora
-----------------------
+The recommended way to install mitmproxy on Windows is to use the installer
+provided at `mitmproxy.org`_. After installation, you'll find shortcuts for
+:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu.
+Both executables are  added to your PATH and can be invoked from the command
+line.
 
-Fedora comes with Python but we need to install pip, python-dev and several libraries.
-This was tested on a fully patched installation of Fedora 24.
+.. note::
+    mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-.. code:: bash
+.. _install-linux:
 
-   sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel
-   sudo pip3 install mitmproxy  # or pip install --user mitmproxy
+Installation on Linux
+---------------------
 
-Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
+The recommended way to run mitmproxy on Linux is to use the pre-built binaries
+provided at `mitmproxy.org`_.
+
+Our pre-built binaries provide you with the latest version of mitmproxy, a
+self-contained Python 3.5 environment and a recent version of OpenSSL that
+supports HTTP/2. Of course, you can also install mitmproxy from source if you
+prefer that (see :ref:`install-advanced`).
+
+.. _install-advanced:
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
+Advanced Installation
+---------------------
 
+.. _install-docker:
+
+Docker Images
+^^^^^^^^^^^^^
+
+You can also use the official mitmproxy images from `DockerHub`_. That being
+said, our portable binaries are just as easy to install and even easier to use. 😊
 
 
 .. _install-arch:
 
-Installation On Arch Linux
---------------------------
+Installation on Arch Linux
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 mitmproxy has been added into the [community] repository. Use pacman to install it:
 
 >>> sudo pacman -S mitmproxy
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
+.. _install-source-ubuntu:
 
-.. _install-macos:
+Installation from Source on Ubuntu
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installation On macOS
-------------------------
+Ubuntu comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Ubuntu 16.04.
 
-You can use Homebrew to install everything:
 .. code:: bash
 
-    brew install mitmproxy
+   sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
 
-Or you can download the pre-built binary packages from `mitmproxy.org`_.
+On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install
+a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take
+a look at pyenv_. Make sure to have an up-to-date version of pip by running
+``pip3 install -U pip``.
 
-Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
+.. _install-source-fedora:
 
+Installation from Source on Fedora
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. _install-windows:
+Fedora comes with Python but we need to install pip3, python3-dev and several
+libraries. This was tested on a fully patched installation of Fedora 24.
 
-Installation On Windows
------------------------
-
-.. note::
-    Please note that mitmdump is the only component of mitmproxy that is supported on Windows at
-    the moment.
+.. code:: bash
 
-    **There is no interactive user interface on Windows.**
+   sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel
+   sudo pip3 install mitmproxy  # or pip3 install --user mitmproxy
 
+Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.
 
-First, install the latest version of Python 3.5 from the `Python website`_.
-If you already have an older version of Python 3.5 installed, make sure to install pip_
-(pip is included in Python by default). If pip aborts with an error, make sure you are using the current version of pip.
 
-.. code:: powershell
 
-    python -m pip install --upgrade pip
+.. _install-source-windows:
 
-Next, add Python and the Python Scripts directory to your **PATH** variable.
-You can do this easily by running the following in powershell:
+🐱💻 Installation from Source on Windows
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-.. code:: powershell
+.. note::
+    mitmproxy's console interface is not supported on Windows, but you can use
+    mitmweb (the web-based interface) and mitmdump.
 
-    [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User")
+First, install the latest version of Python 3.5 or later from the `Python
+website`_. During installation, make sure to select `Add Python to PATH`.
 
 Now, you can install mitmproxy by running
 
 .. code:: powershell
 
-    pip install mitmproxy
+    pip3 install mitmproxy
 
-Once the installation is complete, you can run :ref:`mitmdump` from a command prompt.
 
 
-.. _install-source:
+.. _install-dev-version:
 
-Installation From Source
-------------------------
+Latest Development Version
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you would like to install mitmproxy directly from the master branch on GitHub or would like to
-get set up to contribute to the project, install the dependencies as you would for a regular
-mitmproxy installation. Then see the Hacking_ section of the README on GitHub.
-You can check your system information by running: ``mitmproxy --sysinfo``
+If you would like to install mitmproxy directly from the master branch on GitHub
+or would like to get set up to contribute to the project, install the
+dependencies as you would for a regular installation from source. Then see the
+Hacking_ section of the README on GitHub. You can check your system information
+by running: ``mitmproxy --sysinfo``
 
 
 .. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking
@@ -122,3 +148,4 @@ You can check your system information by running: ``mitmproxy --sysinfo``
 .. _`Python website`: https://www.python.org/downloads/windows/
 .. _pip: https://pip.pypa.io/en/latest/installing.html
 .. _pyenv: https://github.com/yyuu/pyenv
+.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/
diff --git a/docs/mitmweb.rst b/docs/mitmweb.rst
new file mode 100644
index 00000000..e57af059
--- /dev/null
+++ b/docs/mitmweb.rst
@@ -0,0 +1,18 @@
+.. _mitmweb:
+.. program:: mitmweb
+
+mitmweb
+=======
+
+**mitmweb** is mitmproxy's web-based user interface that allows interactive
+examination and modification of HTTP traffic. Like mitmproxy, it differs from
+mitmdump in that all flows are kept in memory, which means that it's intended
+for taking and manipulating small-ish samples.
+
+.. warning::
+
+  Mitmweb is currently in beta. We consider it stable for all features currently
+  exposed in the UI, but it still misses a lot of mitmproxy's features.
+
+
+.. image:: screenshots/mitmweb.png
diff --git a/docs/screenshots/mitmweb.png b/docs/screenshots/mitmweb.png
new file mode 100644
index 00000000..5f0cc925
--- /dev/null
+++ b/docs/screenshots/mitmweb.png
diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst
index 84f2135e..6ec0caaa 100644
--- a/docs/scripting/overview.rst
+++ b/docs/scripting/overview.rst
@@ -85,8 +85,8 @@ and mitmproxy console can place script output in the event buffer.
 
 Here's how this looks:
 
-.. literalinclude:: ../../examples/simple/logging.py
-   :caption: :src:`examples/simple/logging.py`
+.. literalinclude:: ../../examples/simple/log_events.py
+   :caption: :src:`examples/simple/log_events.py`
    :language: python
 
 The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
index aeb154d2..62011903 100644
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -136,7 +136,7 @@ def response(flow):
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
         params = [
-            {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")}
+            {"name": a, "value": b}
             for a, b in flow.request.urlencoded_form.items(multi=True)
         ]
         entry["request"]["postData"] = {
diff --git a/examples/simple/logging.py b/examples/simple/log_events.py
index ab1baf75..ab1baf75 100644
--- a/examples/simple/logging.py
+++ b/examples/simple/log_events.py
diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py
index 0b52918c..f2b5f206 100644
--- a/mitmproxy/addons/serverplayback.py
+++ b/mitmproxy/addons/serverplayback.py
@@ -1,9 +1,10 @@
-import urllib
 import hashlib
+import urllib
+from typing import Any  # noqa
+from typing import List  # noqa
 
-from mitmproxy.utils import strutils
-from mitmproxy import exceptions
 from mitmproxy import ctx
+from mitmproxy import exceptions
 from mitmproxy import io
 
 
@@ -36,17 +37,20 @@ class ServerPlayback:
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key = [str(r.port), str(r.scheme), str(r.method), str(path)]
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
         if not self.options.server_replay_ignore_content:
-            form_contents = r.urlencoded_form or r.multipart_form
-            if self.options.server_replay_ignore_payload_params and form_contents:
-                params = [
-                    strutils.always_bytes(i)
-                    for i in self.options.server_replay_ignore_payload_params
-                ]
-                for p in form_contents.items(multi=True):
-                    if p[0] not in params:
-                        key.append(p)
+            if self.options.server_replay_ignore_payload_params and r.multipart_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.multipart_form.items(multi=True)
+                    if k.decode(errors="replace") not in self.options.server_replay_ignore_payload_params
+                )
+            elif self.options.server_replay_ignore_payload_params and r.urlencoded_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.urlencoded_form.items(multi=True)
+                    if k not in self.options.server_replay_ignore_payload_params
+                )
             else:
                 key.append(str(r.raw_content))
 
diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py
index c7941ad9..9c4bca2f 100644
--- a/mitmproxy/connections.py
+++ b/mitmproxy/connections.py
@@ -6,6 +6,7 @@ import os
 from mitmproxy import stateobject
 from mitmproxy import certs
 from mitmproxy.net import tcp
+from mitmproxy.utils import strutils
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
@@ -22,6 +23,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         timestamp_end: Connection end timestamp
         sni: Server Name Indication sent by client during the TLS handshake
         cipher_name: The current used cipher
+        alpn_proto_negotiated: The negotiated application protocol
         tls_version: TLS version
     """
 
@@ -44,14 +46,22 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         self.timestamp_ssl_setup = None
         self.sni = None
         self.cipher_name = None
+        self.alpn_proto_negotiated = None
         self.tls_version = None
 
     def connected(self):
         return bool(self.connection) and not self.finished
 
     def __repr__(self):
-        return "<ClientConnection: {ssl}{address}>".format(
+        if self.alpn_proto_negotiated:
+            alpn = "[ALPN: {}] ".format(
+                strutils.bytes_to_escaped_str(self.alpn_proto_negotiated)
+            )
+        else:
+            alpn = ""
+        return "<ClientConnection: {ssl}{alpn}{address}>".format(
             ssl="[ssl] " if self.ssl_established else "",
+            alpn=alpn,
             address=repr(self.address)
         )
 
@@ -68,6 +78,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         timestamp_end=float,
         sni=str,
         cipher_name=str,
+        alpn_proto_negotiated=bytes,
         tls_version=str,
     )
 
@@ -97,6 +108,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
             timestamp_ssl_setup=None,
             sni=None,
             cipher_name=None,
+            alpn_proto_negotiated=None,
             tls_version=None,
         ))
 
@@ -109,6 +121,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         else:
             self.sni = None
         self.cipher_name = self.connection.get_cipher_name()
+        self.alpn_proto_negotiated = self.get_alpn_proto_negotiated()
         self.tls_version = self.connection.get_protocol_version_name()
 
     def finish(self):
@@ -128,6 +141,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         ssl_established: True if TLS is established, False otherwise
         cert: The certificate presented by the remote during the TLS handshake
         sni: Server Name Indication sent by the proxy during the TLS handshake
+        alpn_proto_negotiated: The negotiated application protocol
         via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode)
         timestamp_start: Connection start timestamp
         timestamp_tcp_setup: TCP ACK received timestamp
@@ -138,6 +152,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
     def __init__(self, address, source_address=None, spoof_source_address=None):
         tcp.TCPClient.__init__(self, address, source_address, spoof_source_address)
 
+        self.alpn_proto_negotiated = None
         self.via = None
         self.timestamp_start = None
         self.timestamp_end = None
@@ -154,8 +169,15 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
             ssl = "[ssl] "
         else:
             ssl = ""
-        return "<ServerConnection: {ssl}{address}>".format(
+        if self.alpn_proto_negotiated:
+            alpn = "[ALPN: {}] ".format(
+                strutils.bytes_to_escaped_str(self.alpn_proto_negotiated)
+            )
+        else:
+            alpn = ""
+        return "<ServerConnection: {ssl}{alpn}{address}>".format(
             ssl=ssl,
+            alpn=alpn,
             address=repr(self.address)
         )
 
@@ -170,6 +192,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         ssl_established=bool,
         cert=certs.SSLCert,
         sni=str,
+        alpn_proto_negotiated=bytes,
         timestamp_start=float,
         timestamp_tcp_setup=float,
         timestamp_ssl_setup=float,
@@ -189,6 +212,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
             ip_address=dict(address=address, use_ipv6=False),
             cert=None,
             sni=None,
+            alpn_proto_negotiated=None,
             source_address=dict(address=('', 0), use_ipv6=False),
             ssl_established=False,
             timestamp_start=None,
@@ -228,6 +252,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
 
         self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)
         self.sni = sni
+        self.alpn_proto_negotiated = self.get_alpn_proto_negotiated()
         self.timestamp_ssl_setup = time.time()
 
     def finish(self):
diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py
index 9a87a545..9f73693e 100644
--- a/mitmproxy/io_compat.py
+++ b/mitmproxy/io_compat.py
@@ -65,12 +65,19 @@ def convert_017_018(data):
 
 
 def convert_018_019(data):
-    data["version"] = (0, 19)
+    # convert_unicode needs to be called for every dual release and the first py3-only release
+    data = convert_unicode(data)
+
+    data["request"].pop("stickyauth", None)
+    data["request"].pop("stickycookie", None)
     data["client_conn"]["sni"] = None
+    data["client_conn"]["alpn_proto_negotiated"] = None
     data["client_conn"]["cipher_name"] = None
     data["client_conn"]["tls_version"] = None
+    data["server_conn"]["alpn_proto_negotiated"] = None
     data["mode"] = "regular"
     data["metadata"] = dict()
+    data["version"] = (0, 19)
     return data
 
 
@@ -142,7 +149,4 @@ def migrate_flow(flow_data):
             raise ValueError(
                 "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v)
             )
-    # TODO: This should finally be moved in the converter for the first py3-only release.
-    # It's here so that a py2 0.18 dump can be read by py3 0.18 and vice versa.
-    flow_data = convert_unicode(flow_data)
     return flow_data
diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py
index d3d6898d..166f919a 100644
--- a/mitmproxy/net/http/message.py
+++ b/mitmproxy/net/http/message.py
@@ -103,7 +103,11 @@ class Message(serializable.Serializable):
         ce = self.headers.get("content-encoding")
         if ce:
             try:
-                return encoding.decode(self.raw_content, ce)
+                content = encoding.decode(self.raw_content, ce)
+                # A client may illegally specify a byte -> str encoding here (e.g. utf8)
+                if isinstance(content, str):
+                    raise ValueError("Invalid Content-Encoding: {}".format(ce))
+                return content
             except ValueError:
                 if strict:
                     raise
diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py
index c3d85363..7cc4def7 100644
--- a/mitmproxy/net/http/request.py
+++ b/mitmproxy/net/http/request.py
@@ -350,6 +350,8 @@ class Request(message.Message):
         The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         An empty multidict.MultiDictView if the content-type indicates non-form data
         or the content could not be parsed.
+
+        Starting with mitmproxy 1.0, key and value are strings.
         """
         return multidict.MultiDictView(
             self._get_urlencoded_form,
@@ -360,7 +362,7 @@ class Request(message.Message):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
             try:
-                return tuple(mitmproxy.net.http.url.decode(self.content))
+                return tuple(mitmproxy.net.http.url.decode(self.content.decode()))
             except ValueError:
                 pass
         return ()
@@ -381,7 +383,10 @@ class Request(message.Message):
     def multipart_form(self):
         """
         The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
-        None if the content-type indicates non-form data.
+        An empty multidict.MultiDictView if the content-type indicates non-form data
+        or the content could not be parsed.
+
+        Key and value are bytes.
         """
         return multidict.MultiDictView(
             self._get_multipart_form,
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 50d64e17..59de070f 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -1,6 +1,5 @@
 import h2.exceptions
 import time
-import traceback
 import enum
 
 from mitmproxy import connections  # noqa
@@ -431,7 +430,7 @@ class HttpLayer(base.Layer):
             response = http.make_error_response(code, message, headers)
             self.send_response(response)
         except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
-            self.log(traceback.format_exc(), "debug")
+            self.log("Failed to send error response to client: {}".format(message), "debug")
 
     def change_upstream_proxy_server(self, address) -> None:
         # Make set_upstream_proxy_server always available,
diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py
index 41707096..b7548221 100644
--- a/mitmproxy/proxy/protocol/http2.py
+++ b/mitmproxy/proxy/protocol/http2.py
@@ -1,6 +1,5 @@
 import threading
 import time
-import traceback
 import functools
 from typing import Dict, Callable, Any, List  # noqa
 
@@ -328,7 +327,7 @@ class Http2Layer(base.Layer):
 
         try:
             while True:
-                r = tcp.ssl_read_select(conns, 1)
+                r = tcp.ssl_read_select(conns, 0.1)
                 for conn in r:
                     source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn
                     other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn
@@ -358,7 +357,6 @@ class Http2Layer(base.Layer):
                     self._cleanup_streams()
         except Exception as e:  # pragma: no cover
             self.log(repr(e), "info")
-            self.log(traceback.format_exc(), "debug")
             self._kill_all_streams()
 
 
@@ -580,7 +578,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr
     def read_response_body(self, request, response):
         while True:
             try:
-                yield self.response_data_queue.get(timeout=1)
+                yield self.response_data_queue.get(timeout=0.1)
             except queue.Empty:  # pragma: no cover
                 pass
             if self.response_data_finished.is_set():
@@ -624,7 +622,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr
             pass
         except exceptions.ProtocolException as e:  # pragma: no cover
             self.log(repr(e), "info")
-            self.log(traceback.format_exc(), "debug")
         except exceptions.SetServerNotAllowedException as e:  # pragma: no cover
             self.log("Changing the Host server for HTTP/2 connections not allowed: {}".format(e), "info")
         except exceptions.Kill:
diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py
index 2e4c91b0..38f511c4 100644
--- a/mitmproxy/proxy/protocol/http_replay.py
+++ b/mitmproxy/proxy/protocol/http_replay.py
@@ -1,5 +1,3 @@
-import traceback
-
 from mitmproxy import log
 from mitmproxy import controller
 from mitmproxy import exceptions
@@ -107,10 +105,10 @@ class RequestReplayThread(basethread.BaseThread):
                 "log",
                 log.LogEntry("Connection killed", "info")
             )
-        except Exception:
+        except Exception as e:
             self.channel.tell(
                 "log",
-                log.LogEntry(traceback.format_exc(), "error")
+                log.LogEntry(repr(e), "error")
             )
         finally:
             r.first_line_format = first_line_format_backup
diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py
index fc00a633..97018dad 100644
--- a/mitmproxy/proxy/server.py
+++ b/mitmproxy/proxy/server.py
@@ -120,7 +120,6 @@ class ConnectionHandler:
         except exceptions.Kill:
             self.log("Connection killed", "info")
         except exceptions.ProtocolException as e:
-
             if isinstance(e, exceptions.ClientHandshakeException):
                 self.log(
                     "Client Handshake failed. "
@@ -134,7 +133,7 @@ class ConnectionHandler:
             else:
                 self.log(str(e), "warn")
 
-                self.log(traceback.format_exc(), "debug")
+                self.log(repr(e), "debug")
             # If an error propagates to the topmost level,
             # we send an HTTP error response, which is both
             # understandable by HTTP clients and humans.
diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py
index 281047f5..959c9a2c 100644
--- a/mitmproxy/test/tflow.py
+++ b/mitmproxy/test/tflow.py
@@ -72,6 +72,7 @@ def tclient_conn():
         timestamp_end=3,
         sni="address",
         cipher_name="cipher",
+        alpn_proto_negotiated=b"http/1.1",
         tls_version="TLSv1.2",
     ))
     c.reply = controller.DummyReply()
@@ -93,7 +94,8 @@ def tserver_conn():
         timestamp_end=4,
         ssl_established=False,
         sni="address",
-        via=None
+        alpn_proto_negotiated=None,
+        via=None,
     ))
     c.reply = controller.DummyReply()
     return c
diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py
index 7677efe4..571a5e1b 100644
--- a/mitmproxy/tools/console/flowdetailview.py
+++ b/mitmproxy/tools/console/flowdetailview.py
@@ -2,6 +2,7 @@ import urwid
 
 from mitmproxy.tools.console import common, searchable
 from mitmproxy.utils import human
+from mitmproxy.utils import strutils
 
 
 def maybe_timestamp(base, attr):
@@ -31,6 +32,8 @@ def flowdetails(state, flow):
             ["Address", repr(sc.address)],
             ["Resolved Address", repr(sc.ip_address)],
         ]
+        if sc.alpn_proto_negotiated:
+            parts.append(["ALPN", sc.alpn_proto_negotiated])
 
         text.extend(
             common.format_keyvals(parts, key="key", val="text", indent=4)
@@ -75,7 +78,7 @@ def flowdetails(state, flow):
                 parts.append(
                     [
                         "Alt names",
-                        ", ".join(str(x) for x in c.altnames)
+                        ", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames)
                     ]
                 )
             text.extend(
@@ -94,6 +97,8 @@ def flowdetails(state, flow):
             parts.append(["Server Name Indication", cc.sni])
         if cc.cipher_name:
             parts.append(["Cipher Name", cc.cipher_name])
+        if cc.alpn_proto_negotiated:
+            parts.append(["ALPN", cc.alpn_proto_negotiated])
 
         text.extend(
             common.format_keyvals(parts, key="key", val="text", indent=4)
diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py
index 90332627..4e2844a1 100644
--- a/mitmproxy/tools/dump.py
+++ b/mitmproxy/tools/dump.py
@@ -30,12 +30,14 @@ class Options(options.Options):
 
 class DumpMaster(master.Master):
 
-    def __init__(self, options, server):
+    def __init__(self, options, server, with_termlog=True, with_dumper=True):
         master.Master.__init__(self, options, server)
         self.has_errored = False
-        self.addons.add(termlog.TermLog())
+        if with_termlog:
+            self.addons.add(termlog.TermLog())
         self.addons.add(*addons.default_addons())
-        self.addons.add(dumper.Dumper())
+        if with_dumper:
+            self.addons.add(dumper.Dumper())
         # This line is just for type hinting
         self.options = self.options  # type: Options
 
diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py
index ce18c6f0..c0de4c1f 100644
--- a/mitmproxy/tools/web/app.py
+++ b/mitmproxy/tools/web/app.py
@@ -34,6 +34,12 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
         "type": flow.type,
         "modified": flow.modified(),
     }
+    # .alpn_proto_negotiated is bytes, we need to decode that.
+    for conn in "client_conn", "server_conn":
+        if f[conn]["alpn_proto_negotiated"] is None:
+            continue
+        f[conn]["alpn_proto_negotiated"] = \
+            f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace")
     if flow.error:
         f["error"] = flow.error.get_state()
 
diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py
index 68a3db70..e6c15a0a 100644
--- a/mitmproxy/tools/web/master.py
+++ b/mitmproxy/tools/web/master.py
@@ -15,7 +15,7 @@ from mitmproxy.tools.web import app
 
 
 class WebMaster(master.Master):
-    def __init__(self, options, server):
+    def __init__(self, options, server, with_termlog=True):
         super().__init__(options, server)
         self.view = view.View()
         self.view.sig_view_add.connect(self._sig_view_add)
@@ -34,8 +34,9 @@ class WebMaster(master.Master):
             intercept.Intercept(),
             self.view,
             self.events,
-            termlog.TermLog(),
         )
+        if with_termlog:
+            self.addons.add(termlog.TermLog())
         self.app = app.Application(
             self, self.options.wdebug
         )
diff --git a/setup.py b/setup.py
index 2f05eddc..08708b31 100644
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@ setup(
         "cryptography>=1.3, <1.8",
         "cssutils>=1.0.1, <1.1",
         "Flask>=0.10.1, <0.12",
-        "h2>=2.5.0, <3",
+        "h2>=2.5.1, <3",
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=4.0.1, <5",
         "jsbeautifier>=1.6.3, <1.7",
diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018
new file mode 100644
index 00000000..abe8b0b1
--- /dev/null
+++ b/test/mitmproxy/data/dumpfile-018
@@ -0,0 +1,85 @@
+5243:5:error;0:~11:intercepted;5:false!6:marked;5:false!2:id;36:55367415-10f5-4938-b69f-8a523394f947;7:request;396:10:stickyauth;5:false!7:content;0:,15:timestamp_start;18:1482157523.9086578^9:is_replay;5:false!4:path;1:/,4:host;15:www.example.com,17:first_line_format;8:relative;12:stickycookie;5:false!12:http_version;8:HTTP/1.1,6:method;3:GET,4:port;3:443#13:timestamp_end;18:1482157523.9086578^6:scheme;5:https,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]}8:response;1851:6:reason;2:OK,12:http_version;8:HTTP/1.1,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#7:content;1270:<!doctype html>
+<html>
+<head>
+    <title>Example Domain</title>
+
+    <meta charset="utf-8" />
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <style type="text/css">
+    body {
+        background-color: #f0f0f2;
+        margin: 0;
+        padding: 0;
+        font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+        
+    }
+    div {
+        width: 600px;
+        margin: 5em auto;
+        padding: 50px;
+        background-color: #fff;
+        border-radius: 1em;
+    }
+    a:link, a:visited {
+        color: #38488f;
+        text-decoration: none;
+    }
+    @media (max-width: 700px) {
+        body {
+            background-color: #fff;
+        }
+        div {
+            width: auto;
+            margin: 0 auto;
+            border-radius: 0;
+            padding: 1em;
+        }
+    }
+    </style>    
+</head>
+
+<body>
+<div>
+    <h1>Example Domain</h1>
+    <p>This domain is established to be used for illustrative examples in documents. You may use this
+    domain in examples without prior coordination or asking for permission.</p>
+    <p><a href="http://www.iana.org/domains/example">More information...</a></p>
+</div>
+</body>
+</html>
+,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]15:timestamp_start;17:1482157524.361187^}4:type;4:http;11:server_conn;2570:15:ssl_established;4:true!7:address;58:7:address;25:15:www.example.com;3:443#]8:use_ipv6;5:false!}10:ip_address;56:7:address;23:13:93.184.216.34;3:443#]8:use_ipv6;5:false!}3:via;0:~14:source_address;57:7:address;24:12:10.67.53.133;5:52775#]8:use_ipv6;5:false!}13:timestamp_end;0:~4:cert;2122:-----BEGIN CERTIFICATE-----
+MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz
+dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa
+MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML
+TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB
+c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY
+MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u
+Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo
+D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V
+RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3
+jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a
+z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw
+FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW
+5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t
+ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh
+bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud
+DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f
+BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy
+dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt
+aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG
+AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB
+gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy
+dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E
+aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC
+MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q
+6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP
+PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy
+ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI
+l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS
+wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII=
+-----END CERTIFICATE-----
+,15:timestamp_start;18:1482157523.9086578^3:sni;15:www.example.com;19:timestamp_ssl_setup;17:1482157524.260993^19:timestamp_tcp_setup;18:1482157524.0081189^}11:client_conn;216:15:ssl_established;4:true!7:address;53:7:address;20:9:127.0.0.1;5:52774#]8:use_ipv6;5:false!}10:clientcert;0:~13:timestamp_end;0:~15:timestamp_start;18:1482157522.8949482^19:timestamp_ssl_setup;18:1482157523.9086578^}7:version;13:1:0#2:18#1:2#]}
\ No newline at end of file
diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py
index 69d029d9..a001e734 100644
--- a/test/mitmproxy/net/http/test_message.py
+++ b/test/mitmproxy/net/http/test_message.py
@@ -141,6 +141,15 @@ class TestMessageContentEncoding:
         assert r.headers["content-encoding"]
         assert r.get_content(strict=False) == b"foo"
 
+    def test_utf8_as_ce(self):
+        r = tutils.tresp()
+        r.headers["content-encoding"] = "utf8"
+        r.raw_content = b"foo"
+        with tutils.raises(ValueError):
+            assert r.content
+        assert r.headers["content-encoding"]
+        assert r.get_content(strict=False) == b"foo"
+
     def test_cannot_decode(self):
         r = tutils.tresp()
         r.encode("gzip")
diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py
index 9c0ec333..dfa76ba8 100644
--- a/test/mitmproxy/net/http/test_request.py
+++ b/test/mitmproxy/net/http/test_request.py
@@ -255,11 +255,11 @@ class TestRequestUtils:
         assert not request.urlencoded_form
 
         request.headers["Content-Type"] = "application/x-www-form-urlencoded"
-        assert list(request.urlencoded_form.items()) == [(b"foobar", b"baz")]
+        assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
 
     def test_set_urlencoded_form(self):
         request = treq()
-        request.urlencoded_form = [(b'foo', b'bar'), (b'rab', b'oof')]
+        request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')]
         assert request.headers["Content-Type"] == "application/x-www-form-urlencoded"
         assert request.content
 
diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 610c9dad..3930e8df 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -68,11 +68,11 @@ class TestScripts(mastertest.MasterTest):
         f = tflow.tflow(req=tutils.treq(headers=form_header))
         m.request(f)
 
-        assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks"
+        assert f.request.urlencoded_form["mitmproxy"] == "rocks"
 
         f.request.headers["content-type"] = ""
         m.request(f)
-        assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")]
+        assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
 
     def test_modify_querystring(self):
         m, sc = tscript("simple/modify_querystring.py")
diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py
index 0bb9cf15..a787278d 100644
--- a/test/mitmproxy/test_flow_format_compat.py
+++ b/test/mitmproxy/test_flow_format_compat.py
@@ -11,6 +11,14 @@ def test_load():
         assert flows[0].request.url == "https://example.com/"
 
 
+def test_load_018():
+    with open(tutils.test_data.path("mitmproxy/data/dumpfile-018"), "rb") as f:
+        flow_reader = io.FlowReader(f)
+        flows = list(flow_reader.stream())
+        assert len(flows) == 1
+        assert flows[0].request.url == "https://www.example.com/"
+
+
 def test_cannot_convert():
     with open(tutils.test_data.path("mitmproxy/data/dumpfile-010"), "rb") as f:
         flow_reader = io.FlowReader(f)
diff --git a/test/mitmproxy/test_tools_dump.py b/test/mitmproxy/test_tools_dump.py
index 1488f33b..2e64d2d2 100644
--- a/test/mitmproxy/test_tools_dump.py
+++ b/test/mitmproxy/test_tools_dump.py
@@ -11,7 +11,8 @@ from . import mastertest
 class TestDumpMaster(mastertest.MasterTest):
     def mkmaster(self, flt, **options):
         o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options)
-        return dump.DumpMaster(o, proxy.DummyServer())
+        m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False)
+        return m
 
     def test_read(self):
         with tutils.tmpdir() as t:
diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py
index 6cf6ce26..61cf6993 100644
--- a/test/mitmproxy/test_web_app.py
+++ b/test/mitmproxy/test_web_app.py
@@ -19,7 +19,7 @@ def json(resp: httpclient.HTTPResponse):
 class TestApp(tornado.testing.AsyncHTTPTestCase):
     def get_app(self):
         o = options.Options()
-        m = webmaster.WebMaster(o, proxy.DummyServer())
+        m = webmaster.WebMaster(o, proxy.DummyServer(), with_termlog=False)
         f = tflow.tflow(resp=True)
         f.id = "42"
         m.view.add(f)