Merge branch 'master' into contentviews

1 files changed, 250 insertions, 30 deletions

diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py
index a8dc8bb2..4f7c9300 100644
--- a/libmproxy/protocol/tls.py
+++ b/libmproxy/protocol/tls.py
@@ -1,16 +1,210 @@
 from __future__ import (absolute_import, print_function, division)
 
 import struct
+import sys
 
 from construct import ConstructError
+import six
 
 from netlib.tcp import NetLibError, NetLibInvalidCertificateError
 from netlib.http.http1 import HTTP1Protocol
 from ..contrib.tls._constructs import ClientHello
-from ..exceptions import ProtocolException
+from ..exceptions import ProtocolException, TlsException, ClientHandshakeException
 from .base import Layer
 
 
+
+# taken from https://testssl.sh/openssl-rfc.mappping.html
+CIPHER_ID_NAME_MAP = {
+    0x00: 'NULL-MD5',
+    0x01: 'NULL-MD5',
+    0x02: 'NULL-SHA',
+    0x03: 'EXP-RC4-MD5',
+    0x04: 'RC4-MD5',
+    0x05: 'RC4-SHA',
+    0x06: 'EXP-RC2-CBC-MD5',
+    0x07: 'IDEA-CBC-SHA',
+    0x08: 'EXP-DES-CBC-SHA',
+    0x09: 'DES-CBC-SHA',
+    0x0a: 'DES-CBC3-SHA',
+    0x0b: 'EXP-DH-DSS-DES-CBC-SHA',
+    0x0c: 'DH-DSS-DES-CBC-SHA',
+    0x0d: 'DH-DSS-DES-CBC3-SHA',
+    0x0e: 'EXP-DH-RSA-DES-CBC-SHA',
+    0x0f: 'DH-RSA-DES-CBC-SHA',
+    0x10: 'DH-RSA-DES-CBC3-SHA',
+    0x11: 'EXP-EDH-DSS-DES-CBC-SHA',
+    0x12: 'EDH-DSS-DES-CBC-SHA',
+    0x13: 'EDH-DSS-DES-CBC3-SHA',
+    0x14: 'EXP-EDH-RSA-DES-CBC-SHA',
+    0x15: 'EDH-RSA-DES-CBC-SHA',
+    0x16: 'EDH-RSA-DES-CBC3-SHA',
+    0x17: 'EXP-ADH-RC4-MD5',
+    0x18: 'ADH-RC4-MD5',
+    0x19: 'EXP-ADH-DES-CBC-SHA',
+    0x1a: 'ADH-DES-CBC-SHA',
+    0x1b: 'ADH-DES-CBC3-SHA',
+    # 0x1c: ,
+    # 0x1d: ,
+    0x1e: 'KRB5-DES-CBC-SHA',
+    0x1f: 'KRB5-DES-CBC3-SHA',
+    0x20: 'KRB5-RC4-SHA',
+    0x21: 'KRB5-IDEA-CBC-SHA',
+    0x22: 'KRB5-DES-CBC-MD5',
+    0x23: 'KRB5-DES-CBC3-MD5',
+    0x24: 'KRB5-RC4-MD5',
+    0x25: 'KRB5-IDEA-CBC-MD5',
+    0x26: 'EXP-KRB5-DES-CBC-SHA',
+    0x27: 'EXP-KRB5-RC2-CBC-SHA',
+    0x28: 'EXP-KRB5-RC4-SHA',
+    0x29: 'EXP-KRB5-DES-CBC-MD5',
+    0x2a: 'EXP-KRB5-RC2-CBC-MD5',
+    0x2b: 'EXP-KRB5-RC4-MD5',
+    0x2f: 'AES128-SHA',
+    0x30: 'DH-DSS-AES128-SHA',
+    0x31: 'DH-RSA-AES128-SHA',
+    0x32: 'DHE-DSS-AES128-SHA',
+    0x33: 'DHE-RSA-AES128-SHA',
+    0x34: 'ADH-AES128-SHA',
+    0x35: 'AES256-SHA',
+    0x36: 'DH-DSS-AES256-SHA',
+    0x37: 'DH-RSA-AES256-SHA',
+    0x38: 'DHE-DSS-AES256-SHA',
+    0x39: 'DHE-RSA-AES256-SHA',
+    0x3a: 'ADH-AES256-SHA',
+    0x3b: 'NULL-SHA256',
+    0x3c: 'AES128-SHA256',
+    0x3d: 'AES256-SHA256',
+    0x3e: 'DH-DSS-AES128-SHA256',
+    0x3f: 'DH-RSA-AES128-SHA256',
+    0x40: 'DHE-DSS-AES128-SHA256',
+    0x41: 'CAMELLIA128-SHA',
+    0x42: 'DH-DSS-CAMELLIA128-SHA',
+    0x43: 'DH-RSA-CAMELLIA128-SHA',
+    0x44: 'DHE-DSS-CAMELLIA128-SHA',
+    0x45: 'DHE-RSA-CAMELLIA128-SHA',
+    0x46: 'ADH-CAMELLIA128-SHA',
+    0x62: 'EXP1024-DES-CBC-SHA',
+    0x63: 'EXP1024-DHE-DSS-DES-CBC-SHA',
+    0x64: 'EXP1024-RC4-SHA',
+    0x65: 'EXP1024-DHE-DSS-RC4-SHA',
+    0x66: 'DHE-DSS-RC4-SHA',
+    0x67: 'DHE-RSA-AES128-SHA256',
+    0x68: 'DH-DSS-AES256-SHA256',
+    0x69: 'DH-RSA-AES256-SHA256',
+    0x6a: 'DHE-DSS-AES256-SHA256',
+    0x6b: 'DHE-RSA-AES256-SHA256',
+    0x6c: 'ADH-AES128-SHA256',
+    0x6d: 'ADH-AES256-SHA256',
+    0x80: 'GOST94-GOST89-GOST89',
+    0x81: 'GOST2001-GOST89-GOST89',
+    0x82: 'GOST94-NULL-GOST94',
+    0x83: 'GOST2001-GOST89-GOST89',
+    0x84: 'CAMELLIA256-SHA',
+    0x85: 'DH-DSS-CAMELLIA256-SHA',
+    0x86: 'DH-RSA-CAMELLIA256-SHA',
+    0x87: 'DHE-DSS-CAMELLIA256-SHA',
+    0x88: 'DHE-RSA-CAMELLIA256-SHA',
+    0x89: 'ADH-CAMELLIA256-SHA',
+    0x8a: 'PSK-RC4-SHA',
+    0x8b: 'PSK-3DES-EDE-CBC-SHA',
+    0x8c: 'PSK-AES128-CBC-SHA',
+    0x8d: 'PSK-AES256-CBC-SHA',
+    # 0x8e: ,
+    # 0x8f: ,
+    # 0x90: ,
+    # 0x91: ,
+    # 0x92: ,
+    # 0x93: ,
+    # 0x94: ,
+    # 0x95: ,
+    0x96: 'SEED-SHA',
+    0x97: 'DH-DSS-SEED-SHA',
+    0x98: 'DH-RSA-SEED-SHA',
+    0x99: 'DHE-DSS-SEED-SHA',
+    0x9a: 'DHE-RSA-SEED-SHA',
+    0x9b: 'ADH-SEED-SHA',
+    0x9c: 'AES128-GCM-SHA256',
+    0x9d: 'AES256-GCM-SHA384',
+    0x9e: 'DHE-RSA-AES128-GCM-SHA256',
+    0x9f: 'DHE-RSA-AES256-GCM-SHA384',
+    0xa0: 'DH-RSA-AES128-GCM-SHA256',
+    0xa1: 'DH-RSA-AES256-GCM-SHA384',
+    0xa2: 'DHE-DSS-AES128-GCM-SHA256',
+    0xa3: 'DHE-DSS-AES256-GCM-SHA384',
+    0xa4: 'DH-DSS-AES128-GCM-SHA256',
+    0xa5: 'DH-DSS-AES256-GCM-SHA384',
+    0xa6: 'ADH-AES128-GCM-SHA256',
+    0xa7: 'ADH-AES256-GCM-SHA384',
+    0x5600: 'TLS_FALLBACK_SCSV',
+    0xc001: 'ECDH-ECDSA-NULL-SHA',
+    0xc002: 'ECDH-ECDSA-RC4-SHA',
+    0xc003: 'ECDH-ECDSA-DES-CBC3-SHA',
+    0xc004: 'ECDH-ECDSA-AES128-SHA',
+    0xc005: 'ECDH-ECDSA-AES256-SHA',
+    0xc006: 'ECDHE-ECDSA-NULL-SHA',
+    0xc007: 'ECDHE-ECDSA-RC4-SHA',
+    0xc008: 'ECDHE-ECDSA-DES-CBC3-SHA',
+    0xc009: 'ECDHE-ECDSA-AES128-SHA',
+    0xc00a: 'ECDHE-ECDSA-AES256-SHA',
+    0xc00b: 'ECDH-RSA-NULL-SHA',
+    0xc00c: 'ECDH-RSA-RC4-SHA',
+    0xc00d: 'ECDH-RSA-DES-CBC3-SHA',
+    0xc00e: 'ECDH-RSA-AES128-SHA',
+    0xc00f: 'ECDH-RSA-AES256-SHA',
+    0xc010: 'ECDHE-RSA-NULL-SHA',
+    0xc011: 'ECDHE-RSA-RC4-SHA',
+    0xc012: 'ECDHE-RSA-DES-CBC3-SHA',
+    0xc013: 'ECDHE-RSA-AES128-SHA',
+    0xc014: 'ECDHE-RSA-AES256-SHA',
+    0xc015: 'AECDH-NULL-SHA',
+    0xc016: 'AECDH-RC4-SHA',
+    0xc017: 'AECDH-DES-CBC3-SHA',
+    0xc018: 'AECDH-AES128-SHA',
+    0xc019: 'AECDH-AES256-SHA',
+    0xc01a: 'SRP-3DES-EDE-CBC-SHA',
+    0xc01b: 'SRP-RSA-3DES-EDE-CBC-SHA',
+    0xc01c: 'SRP-DSS-3DES-EDE-CBC-SHA',
+    0xc01d: 'SRP-AES-128-CBC-SHA',
+    0xc01e: 'SRP-RSA-AES-128-CBC-SHA',
+    0xc01f: 'SRP-DSS-AES-128-CBC-SHA',
+    0xc020: 'SRP-AES-256-CBC-SHA',
+    0xc021: 'SRP-RSA-AES-256-CBC-SHA',
+    0xc022: 'SRP-DSS-AES-256-CBC-SHA',
+    0xc023: 'ECDHE-ECDSA-AES128-SHA256',
+    0xc024: 'ECDHE-ECDSA-AES256-SHA384',
+    0xc025: 'ECDH-ECDSA-AES128-SHA256',
+    0xc026: 'ECDH-ECDSA-AES256-SHA384',
+    0xc027: 'ECDHE-RSA-AES128-SHA256',
+    0xc028: 'ECDHE-RSA-AES256-SHA384',
+    0xc029: 'ECDH-RSA-AES128-SHA256',
+    0xc02a: 'ECDH-RSA-AES256-SHA384',
+    0xc02b: 'ECDHE-ECDSA-AES128-GCM-SHA256',
+    0xc02c: 'ECDHE-ECDSA-AES256-GCM-SHA384',
+    0xc02d: 'ECDH-ECDSA-AES128-GCM-SHA256',
+    0xc02e: 'ECDH-ECDSA-AES256-GCM-SHA384',
+    0xc02f: 'ECDHE-RSA-AES128-GCM-SHA256',
+    0xc030: 'ECDHE-RSA-AES256-GCM-SHA384',
+    0xc031: 'ECDH-RSA-AES128-GCM-SHA256',
+    0xc032: 'ECDH-RSA-AES256-GCM-SHA384',
+    0xcc13: 'ECDHE-RSA-CHACHA20-POLY1305',
+    0xcc14: 'ECDHE-ECDSA-CHACHA20-POLY1305',
+    0xcc15: 'DHE-RSA-CHACHA20-POLY1305',
+    0xff00: 'GOST-MD5',
+    0xff01: 'GOST-GOST94',
+    0xff02: 'GOST-GOST89MAC',
+    0xff03: 'GOST-GOST89STREAM',
+    0x010080: 'RC4-MD5',
+    0x020080: 'EXP-RC4-MD5',
+    0x030080: 'RC2-CBC-MD5',
+    0x040080: 'EXP-RC2-CBC-MD5',
+    0x050080: 'IDEA-CBC-MD5',
+    0x060040: 'DES-CBC-MD5',
+    0x0700c0: 'DES-CBC3-MD5',
+    0x080080: 'RC4-64-MD5',
+}
+
+
 def is_tls_record_magic(d):
     """
     Returns:
@@ -47,8 +241,8 @@ class TlsLayer(Layer):
             If so, we first connect to the server and then to the client.
             If not, we only connect to the client and do the server_ssl lazily on a Connect message.
 
-        An additional complexity is that establish ssl with the server may require a SNI value from the client.
-        In an ideal world, we'd do the following:
+        An additional complexity is that establish ssl with the server may require a SNI value from
+        the client. In an ideal world, we'd do the following:
             1. Start the SSL handshake with the client
             2. Check if the client sends a SNI.
             3. Pause the client handshake, establish SSL with the server.
@@ -100,11 +294,11 @@ class TlsLayer(Layer):
         while len(client_hello) < client_hello_size:
             record_header = self.client_conn.rfile.peek(offset + 5)[offset:]
             if not is_tls_record_magic(record_header) or len(record_header) != 5:
-                raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header)
+                raise TlsException('Expected TLS record, got "%s" instead.' % record_header)
             record_size = struct.unpack("!H", record_header[3:])[0] + 5
             record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:]
             if len(record_body) != record_size - 5:
-                raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
+                raise TlsException("Unexpected EOF in TLS handshake: %s" % record_body)
             client_hello += record_body
             offset += record_size
             client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4
@@ -127,6 +321,8 @@ class TlsLayer(Layer):
             self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug")
             return
 
+        self.client_ciphers = client_hello.cipher_suites.cipher_suites
+
         for extension in client_hello.extensions:
             if extension.type == 0x00:
                 if len(extension.server_names) != 1 or extension.server_names[0].type != 0:
@@ -146,18 +342,11 @@ class TlsLayer(Layer):
         if self._server_tls and not self.server_conn.tls_established:
             self._establish_tls_with_server()
 
-    def reconnect(self):
-        self.ctx.reconnect()
-        if self._server_tls and not self.server_conn.tls_established:
-            self._establish_tls_with_server()
-
-    def set_server(self, address, server_tls=None, sni=None, depth=1):
-        if depth == 1 and server_tls is not None:
-            self.ctx.set_server(address, None, None, 1)
+    def set_server(self, address, server_tls=None, sni=None):
+        if server_tls is not None:
             self._sni_from_server_change = sni
             self._server_tls = server_tls
-        else:
-            self.ctx.set_server(address, server_tls, sni, depth)
+        self.ctx.set_server(address, None, None)
 
     @property
     def sni_for_server_connection(self):
@@ -201,7 +390,7 @@ class TlsLayer(Layer):
                 self._establish_tls_with_client()
             except:
                 pass
-            raise e
+            six.reraise(*sys.exc_info())
 
         self._establish_tls_with_client()
 
@@ -219,8 +408,22 @@ class TlsLayer(Layer):
                 chain_file=chain_file,
                 alpn_select_callback=self.__alpn_select_callback,
             )
+            # Some TLS clients will not fail the handshake,
+            # but will immediately throw an "unexpected eof" error on the first read.
+            # The reason for this might be difficult to find, so we try to peek here to see if it
+            # raises ann error.
+            self.client_conn.rfile.peek(1)
         except NetLibError as e:
-            raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e)
+            six.reraise(
+                ClientHandshakeException,
+                ClientHandshakeException(
+                    "Cannot establish TLS with client (sni: {sni}): {e}".format(
+                        sni=self.client_sni, e=repr(e)
+                    ),
+                    self.client_sni or repr(self.server_conn.address)
+                ),
+                sys.exc_info()[2]
+            )
 
     def _establish_tls_with_server(self):
         self.log("Establish TLS with server", "debug")
@@ -230,9 +433,19 @@ class TlsLayer(Layer):
             # and mitmproxy would enter TCP passthrough mode, which we want to avoid.
             deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy")
             if self.client_alpn_protocols:
-                alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols)
+                alpn = [x for x in self.client_alpn_protocols if not deprecated_http2_variant(x)]
             else:
                 alpn = None
+            if alpn and "h2" in alpn and not self.config.http2 :
+                alpn.remove("h2")
+
+            ciphers_server = self.config.ciphers_server
+            if not ciphers_server:
+                ciphers_server = []
+                for id in self.client_ciphers:
+                    if id in CIPHER_ID_NAME_MAP.keys():
+                        ciphers_server.append(CIPHER_ID_NAME_MAP[id])
+                ciphers_server = ':'.join(ciphers_server)
 
             self.server_conn.establish_ssl(
                 self.config.clientcerts,
@@ -242,7 +455,7 @@ class TlsLayer(Layer):
                 verify_options=self.config.openssl_verification_mode_server,
                 ca_path=self.config.openssl_trusted_cadir_server,
                 ca_pemfile=self.config.openssl_trusted_ca_server,
-                cipher_list=self.config.ciphers_server,
+                cipher_list=ciphers_server,
                 alpn_protos=alpn,
             )
             tls_cert_err = self.server_conn.ssl_verification_error
@@ -259,17 +472,25 @@ class TlsLayer(Layer):
                 (tls_cert_err['depth'], tls_cert_err['errno']),
                 "error")
             self.log("Aborting connection attempt", "error")
-            raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
-                address=repr(self.server_conn.address),
-                sni=self.sni_for_server_connection,
-                e=repr(e),
-            ), e)
+            six.reraise(
+                TlsException,
+                TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
+                    address=repr(self.server_conn.address),
+                    sni=self.sni_for_server_connection,
+                    e=repr(e),
+                )),
+                sys.exc_info()[2]
+            )
         except NetLibError as e:
-            raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
-                address=repr(self.server_conn.address),
-                sni=self.sni_for_server_connection,
-                e=repr(e),
-            ), e)
+            six.reraise(
+                TlsException,
+                TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
+                    address=repr(self.server_conn.address),
+                    sni=self.sni_for_server_connection,
+                    e=repr(e),
+                )),
+                sys.exc_info()[2]
+            )
 
         self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug")
 
@@ -294,5 +515,4 @@ class TlsLayer(Layer):
         if self._sni_from_server_change:
             sans.add(self._sni_from_server_change)
 
-        sans.discard(host)
         return self.config.certstore.get_cert(host, list(sans))