diff options
87 files changed, 1088 insertions, 751 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 3ef985be..6891f1b3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -79,6 +79,7 @@ deploy_script: ($Env:TOXENV -match "py35") -and (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { + tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts tox -e rtool -- upload-snapshot --bdist --wheel --installer } diff --git a/.travis.yml b/.travis.yml index a29d0c75..b7504097 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,7 @@ after_success: - | if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then + tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts tox -e rtool -- upload-snapshot --bdist fi diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index 632783a7..e28934ab 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -33,7 +33,7 @@ parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$") class Rerouter: def request(self, flow): - if flow.client_conn.ssl_established: + if flow.client_conn.tls_established: flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() port = 443 diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 21bcc341..66a81a7d 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -58,8 +58,8 @@ def response(flow): connect_time = (flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start) - if flow.server_conn.timestamp_ssl_setup is not None: - ssl_time = (flow.server_conn.timestamp_ssl_setup - + if flow.server_conn.timestamp_tls_setup is not None: + ssl_time = (flow.server_conn.timestamp_tls_setup - flow.server_conn.timestamp_tcp_setup) SERVERS_SEEN.add(flow.server_conn) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 3fd96669..2dd488b9 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -35,6 +35,9 @@ class ClientPlayback: """ Replay requests from flows. """ + for f in flows: + if f.live: + raise exceptions.CommandError("Can't replay live flow.") self.flows = list(flows) ctx.log.alert("Replaying %s flows." % len(self.flows)) ctx.master.addons.trigger("update", []) diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index f4b560e8..d684b8c7 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -43,7 +43,7 @@ def extract(cut: str, f: flow.Flow) -> typing.Union[str, bytes]: return part elif isinstance(part, bool): return "true" if part else "false" - elif isinstance(part, certs.SSLCert): + elif isinstance(part, certs.Cert): return part.to_pem().decode("ascii") current = part return str(current or "") diff --git a/mitmproxy/addons/onboardingapp/static/images/favicon.ico b/mitmproxy/addons/onboardingapp/static/images/favicon.ico Binary files differnew file mode 100644 index 00000000..3c3b891c --- /dev/null +++ b/mitmproxy/addons/onboardingapp/static/images/favicon.ico diff --git a/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png b/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png Binary files differnew file mode 100644 index 00000000..f9397d1e --- /dev/null +++ b/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css index b390976a..969bd62b 100644 --- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css +++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css @@ -1,8 +1,6 @@ - #certbank div { text-align: center; - - + padding-top: 20px; } .fronttable { @@ -40,7 +38,6 @@ section { .masthead { padding: 50px 0 60px; text-align: center; - } .header { diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index fc6213ea..38aa27ed 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -4,59 +4,135 @@ <script> function changeTo(device) { if (device == "apple") { - var text = `<h3>Apple: How to install on macOS / OSX</h3> - <ul> - <li>Double-click the PEM file</li> - <li>The "Keychain Access" applications opens</li> - <li>Find the new certificate "mitmproxy" in the list</li> - <li>Double-click the "mitmproxy" entry</li> - <li>A dialog window openes up</li> - <li>Change "Secure Socket Layer (SSL)" to "Always Trust"</li> - <li>Close the dialog window (and enter your password if prompted)</li> - <li>For iOS version 10.3 or up, you need to make sure mitmproxy is enabled in<br> - Certificate Trust Settings, you can check it by going to<br> - Settings > General > About > Certificate Trust Settings</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div> + <div class="col-md-4"> + <h3 class="text-center">How to install on macOS</h3> + <ul class="left"> + <li>Double-click the PEM file</li> + <li>The "Keychain Access" applications opens</li> + <li>Find the new certificate "mitmproxy" in the list</li> + <li>Double-click the "mitmproxy" entry</li> + <li>A dialog window openes up</li> + <li>Change "Secure Socket Layer (SSL)" to "Always Trust"</li> + <li>Close the dialog window (and enter your password if prompted)</li> + <li>Done!</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on browsers</h3> + <ul> + <li>Safari on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> + <li>Chrome on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> + <li>Firefox on macOS has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on iOS v10.3</h3> + <ul> + <li>After certificate installation, open Settings</li> + <li>Navigate to General and then About</li> + <li>Select Certificate Trust Settings</li> + <li>Each root that has been installed via a profile will be listed below the heading Enable Full Trust For Root Certificates. Toggle mitmproxy on</li> + <li>Done!</li> + </div> + </div> + </div>`; } else if (device == "windows") { - var text = `<h3>Windows: How to install on Windows</h3> - <ul> - <li>Double-click the P12 file</li> - <li>Select Store Location for Current User and click Next</li> - <li>Click Next</li> - <li>Leave the Password column blank and click Next</li> - <li>Select Place all certificates in the following store</li> - <li>Click Browse and select Trusted Root Certification Authorities</li> - <li>Click Next and then click Finish</li> - <li>Click Yes if prompted for confirmation</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Windows</h3> + <ul> + <li>Double-click the P12 file</li> + <li>Select Store Location for Current User and click Next</li> + <li>Click Next</li> + <li>Leave the Password column blank and click Next</li> + <li>Select Place all certificates in the following store</li> + <li>Click Browse and select Trusted Root Certification Authorities</li> + <li>Click Next and then click Finish</li> + <li>Click Yes if prompted for confirmation</li> + <li>Done!</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on browsers</h3> + <ul> + <li>Edge and IE use the Windows CA store. So installing our CA in the system is enough.</li> + <li>Chrome on Windows uses the Windows CA store. So installing our CA in the system is enough.</li> + <li>Firefox on Windows has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Windows (Automated)</h3> + <ul> + <li> >>> certutil.exe -importpfx Root mitmproxy-ca-cert.p12 </li> + <li> To know more click <a href="https://technet.microsoft.com/en-us/library/cc732443.aspx">HERE</a> </li> + </ul> + </div> + </div> + </div>`; } else if (device == "android") { - var text = `<h3>Android: How to install on Android</h3> - <ul> - <li>Open your device's Settings app</li> - <li>Under "Credential storage," tap Install from storage</li> - <li>Under "Open from," tap where you saved the certificate</li> - <li>Tap the file</li> - <li>If prompted, enter the key store password and tap OK</li> - <li>Type a name for the certificate</li> - <li>Pick VPN and apps</li> - <li>Tap OK</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Android</h3> + <ul> + <li>Open your device's Settings app</li> + <li>Under "Credential storage," tap Install from storage</li> + <li>Under "Open from," tap where you saved the certificate</li> + <li>Tap the file</li> + <li>If prompted, enter the key store password and tap OK</li> + <li>Type a name for the certificate</li> + <li>Pick VPN and apps</li> + <li>Tap OK</li> + <li>Done!</li> + </ul> + </div> + </div> + </div>`; } else if (device == "asterisk") { - var text = ""; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Chrome on Debian/Ubuntu</h3> + <ul> + <li>Using Chrome, hit a page on your server via HTTPS and continue past the red warning page (assuming you haven't done this already)</li> + <li>Open up Chrome Settings > Show advanced settings > HTTPS/SSL > Manage Certificates</li> + <li>Click the Authorities tab and scroll down to find your certificate under the Organization Name that you gave to the certificate</li> + <li>Select it, click Edit (NOTE: in recent versions of Chrome, the button is now "Advanced" instead of "Edit"), check all the boxes and click OK. You may have to restart Chrome</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Chrome on Linux</h3> + <ul> + <li>Open Developer Tools > Security, and select View certificate</li> + <li>Click the Details tab > Export. Choose PKCS #7, single certificate as the file format</li> + <li>Then follow my original instructions to get to the Manage Certificates page. Click the Authorities tab > Import and choose the file to which you exported the certificate, and make sure to choose PKCS #7, single certificate as the file type</li> + <li>If prompted certification store, choose Trusted Root Certificate Authorities</li> + <li>Check all boxes and click OK. Restart Chrome</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Ubuntu (Manually)</h3> + <ul> + <li>Create a directory for extra CA certificates in /usr/share/ca-certificates: <div class="text-muted">$ sudo mkdir /usr/share/ca-certificates/extra<div></li> + <li>Copy the CA mitmproxy.crt file to this directory: <div class="text-muted">$ sudo cp mitmproxy.crt /usr/share/ca-certificates/extra/mitmproxy.crt<div></li> + <li>Let Ubuntu add the mitmproxy.crt file's path relative to /usr/share/ca-certificates to /etc/ca-certificates.conf: <div class="text-muted">$ sudo dpkg-reconfigure ca-certificates</div></li> + <li>In case of a .pem file on Ubuntu, it must first be converted to a .crt file: <div class="text-muted">$ openssl x509 -in foo.pem -inform PEM -out foo.crt</div></li> + </ul> + </div> + </div> + </div>`; } document.getElementById("dynamic").innerHTML = text; } </script> -<center> -<h2> Click to install your mitmproxy certificate: </h2> -</center> +<h2 class="text-center"> Click to install your mitmproxy certificate </h2> <div id="certbank" class="row"> <div class="col-md-3"> <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html index 8726a788..f6e1b286 100644 --- a/mitmproxy/addons/onboardingapp/templates/layout.html +++ b/mitmproxy/addons/onboardingapp/templates/layout.html @@ -12,20 +12,23 @@ <link href="/static/bootstrap.min.css" rel="stylesheet"> <link href="/static/mitmproxy.css" rel="stylesheet"> <link href="/static/fontawesome/css/font-awesome.min.css" rel="stylesheet"> + <link rel="icon" href="/static/images/favicon.ico" type="image/x-icon"/> </head> <body> <div class="navbar navbar-default" role="navigation"> <div class="container"> <div class="navbar-header"> - <a class="navbar-brand" href="#">mitmproxy</a> + <a class="navbar-brand" href="#"> + <img height="20px" src="static/images/mitmproxy-long.png"/> + </a> </div> </div> </div> <div class="container"> - {% block content %} - {% end %} + {% block content %} + {% end %} </div> </body> diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 64233e88..dc99d5cc 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -146,14 +146,14 @@ class ProxyAuth: ) elif ctx.options.proxyauth.startswith("ldap"): parts = ctx.options.proxyauth.split(':') - security = parts[0] - ldap_server = parts[1] - dn_baseauth = parts[2] - password_baseauth = parts[3] if len(parts) != 5: raise exceptions.OptionsError( "Invalid ldap specification" ) + security = parts[0] + ldap_server = parts[1] + dn_baseauth = parts[2] + password_baseauth = parts[3] if security == "ldaps": server = ldap3.Server(ldap_server, use_ssl=True) elif security == "ldap": diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 3a9f2c19..2a7e2d09 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -24,7 +24,8 @@ class TermLog: click.secho( e.msg, file=outfile, - fg=dict(error="red", warn="yellow").get(e.level), + fg=dict(error="red", warn="yellow", + alert="magenta").get(e.level), dim=(e.level == "debug"), err=(e.level == "error") ) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 3a15fd3e..c3fb4b88 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -238,7 +238,7 @@ class View(collections.Sequence): @command.command("view.order.options") def order_options(self) -> typing.Sequence[str]: """ - Choices supported by the console_order option. + Choices supported by the view_order option. """ return list(sorted(self.orders.keys())) @@ -441,7 +441,10 @@ class View(collections.Sequence): @command.command("view.create") def create(self, method: str, url: str) -> None: - req = http.HTTPRequest.make(method.upper(), url) + try: + req = http.HTTPRequest.make(method.upper(), url) + except ValueError as e: + raise exceptions.CommandError("Invalid URL: %s" % e) c = connections.ClientConnection.make_dummy(("", 0)) s = connections.ServerConnection.make_dummy((req.host, req.port)) f = http.HTTPFlow(c, s) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index c29d67f3..4e10529a 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -112,7 +112,7 @@ def dummy_cert(privkey, cacert, commonname, sans): [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)]) cert.set_pubkey(cacert.get_pubkey()) cert.sign(privkey, "sha256") - return SSLCert(cert) + return Cert(cert) class CertStoreEntry: @@ -249,7 +249,7 @@ class CertStore: def add_cert_file(self, spec: str, path: str) -> None: with open(path, "rb") as f: raw = f.read() - cert = SSLCert( + cert = Cert( OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, raw)) @@ -345,7 +345,7 @@ class _GeneralNames(univ.SequenceOf): constraint.ValueSizeConstraint(1, 1024) -class SSLCert(serializable.Serializable): +class Cert(serializable.Serializable): def __init__(self, cert): """ @@ -436,7 +436,7 @@ class SSLCert(serializable.Serializable): Returns: All DNS altnames. """ - # tcp.TCPClient.convert_to_ssl assumes that this property only contains DNS altnames for hostname verification. + # tcp.TCPClient.convert_to_tls assumes that this property only contains DNS altnames for hostname verification. altnames = [] for i in range(self.x509.get_extension_count()): ext = self.x509.get_extension(i) diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 01721a71..86565b7b 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -1,11 +1,13 @@ import time import os +import typing import uuid -from mitmproxy import stateobject +from mitmproxy import stateobject, exceptions from mitmproxy import certs from mitmproxy.net import tcp +from mitmproxy.net import tls from mitmproxy.utils import strutils @@ -16,16 +18,17 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): Attributes: address: Remote address - ssl_established: True if TLS is established, False otherwise + tls_established: True if TLS is established, False otherwise clientcert: The TLS client certificate mitmcert: The MITM'ed TLS server certificate presented to the client timestamp_start: Connection start timestamp - timestamp_ssl_setup: TLS established timestamp + timestamp_tls_setup: TLS established timestamp timestamp_end: Connection end timestamp sni: Server Name Indication sent by client during the TLS handshake cipher_name: The current used cipher alpn_proto_negotiated: The negotiated application protocol tls_version: TLS version + tls_extensions: TLS ClientHello extensions """ def __init__(self, client_connection, address, server): @@ -40,23 +43,24 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.rfile = None self.address = None self.clientcert = None - self.ssl_established = None + self.tls_established = None self.id = str(uuid.uuid4()) self.mitmcert = None self.timestamp_start = time.time() self.timestamp_end = None - self.timestamp_ssl_setup = None + self.timestamp_tls_setup = None self.sni = None self.cipher_name = None self.alpn_proto_negotiated = None self.tls_version = None + self.tls_extensions = None def connected(self): return bool(self.connection) and not self.finished def __repr__(self): - if self.ssl_established: + if self.tls_established: tls = "[{}] ".format(self.tls_version) else: tls = "" @@ -83,27 +87,20 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): def __hash__(self): return hash(self.id) - @property - def tls_established(self): - return self.ssl_established - - @tls_established.setter - def tls_established(self, value): - self.ssl_established = value - _stateobject_attributes = dict( id=str, address=tuple, - ssl_established=bool, - clientcert=certs.SSLCert, - mitmcert=certs.SSLCert, + tls_established=bool, + clientcert=certs.Cert, + mitmcert=certs.Cert, timestamp_start=float, - timestamp_ssl_setup=float, + timestamp_tls_setup=float, timestamp_end=float, sni=str, cipher_name=str, alpn_proto_negotiated=bytes, tls_version=str, + tls_extensions=typing.List[typing.Tuple[int, bytes]], ) def send(self, message): @@ -125,19 +122,29 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): address=address, clientcert=None, mitmcert=None, - ssl_established=False, + tls_established=False, timestamp_start=None, timestamp_end=None, - timestamp_ssl_setup=None, + timestamp_tls_setup=None, sni=None, cipher_name=None, alpn_proto_negotiated=None, tls_version=None, + tls_extensions=None, )) - def convert_to_ssl(self, cert, *args, **kwargs): - super().convert_to_ssl(cert, *args, **kwargs) - self.timestamp_ssl_setup = time.time() + def convert_to_tls(self, cert, *args, **kwargs): + # Unfortunately OpenSSL provides no way to expose all TLS extensions, so we do this dance + # here and use our Kaitai parser. + try: + client_hello = tls.ClientHello.from_file(self.rfile) + except exceptions.TlsProtocolException: # pragma: no cover + pass # if this fails, we don't want everything to go down. + else: + self.tls_extensions = client_hello.extensions + + super().convert_to_tls(cert, *args, **kwargs) + self.timestamp_tls_setup = time.time() self.mitmcert = cert sni = self.connection.get_servername() if sni: @@ -162,7 +169,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): address: Remote address. Can be both a domain or an IP address. ip_address: Resolved remote IP address. source_address: Local IP address or client's source IP address. - ssl_established: True if TLS is established, False otherwise + tls_established: True if TLS is established, False otherwise cert: The certificate presented by the remote during the TLS handshake sni: Server Name Indication sent by the proxy during the TLS handshake alpn_proto_negotiated: The negotiated application protocol @@ -170,7 +177,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode) timestamp_start: Connection start timestamp timestamp_tcp_setup: TCP ACK received timestamp - timestamp_ssl_setup: TLS established timestamp + timestamp_tls_setup: TLS established timestamp timestamp_end: Connection end timestamp """ @@ -184,15 +191,15 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None + self.timestamp_tls_setup = None def connected(self): return bool(self.connection) and not self.finished def __repr__(self): - if self.ssl_established and self.sni: + if self.tls_established and self.sni: tls = "[{}: {}] ".format(self.tls_version or "TLS", self.sni) - elif self.ssl_established: + elif self.tls_established: tls = "[{}] ".format(self.tls_version or "TLS") else: tls = "" @@ -217,27 +224,19 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __hash__(self): return hash(self.id) - @property - def tls_established(self): - return self.ssl_established - - @tls_established.setter - def tls_established(self, value): - self.ssl_established = value - _stateobject_attributes = dict( id=str, address=tuple, ip_address=tuple, source_address=tuple, - ssl_established=bool, - cert=certs.SSLCert, + tls_established=bool, + cert=certs.Cert, sni=str, alpn_proto_negotiated=bytes, tls_version=str, timestamp_start=float, timestamp_tcp_setup=float, - timestamp_ssl_setup=float, + timestamp_tls_setup=float, timestamp_end=float, ) @@ -258,10 +257,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): alpn_proto_negotiated=None, tls_version=None, source_address=('', 0), - ssl_established=False, + tls_established=False, timestamp_start=None, timestamp_tcp_setup=None, - timestamp_ssl_setup=None, + timestamp_tls_setup=None, timestamp_end=None, via=None )) @@ -277,7 +276,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.wfile.write(message) self.wfile.flush() - def establish_ssl(self, clientcerts, sni, **kwargs): + def establish_tls(self, clientcerts, sni, **kwargs): if sni and not isinstance(sni, str): raise ValueError("sni must be str, not " + type(sni).__name__) clientcert = None @@ -291,11 +290,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) + self.convert_to_tls(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.alpn_proto_negotiated = self.get_alpn_proto_negotiated() self.tls_version = self.connection.get_protocol_version_name() - self.timestamp_ssl_setup = time.time() + self.timestamp_tls_setup = time.time() def finish(self): tcp.TCPClient.finish(self) diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 97740eea..bdab1e99 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -43,9 +43,11 @@ def format_dict( ) -> typing.Iterator[TViewLine]: """ Helper function that transforms the given dictionary into a list of + [ ("key", key ) ("value", value) - tuples, where key is padded to a uniform width. + ] + entries, where key is padded to a uniform width. """ max_key_len = max(len(k) for k in d.keys()) max_key_len = min(max_key_len, KEY_MAX) diff --git a/mitmproxy/contrib/kaitaistruct/gif.py b/mitmproxy/contrib/kaitaistruct/gif.py index 820df568..76d7fc16 100644 --- a/mitmproxy/contrib/kaitaistruct/gif.py +++ b/mitmproxy/contrib/kaitaistruct/gif.py @@ -35,9 +35,11 @@ class Gif(KaitaiStruct): self.global_color_table = self._root.ColorTable(io, self, self._root) self.blocks = [] - while not self._io.is_eof(): - self.blocks.append(self._root.Block(self._io, self, self._root)) - + while True: + _ = self._root.Block(self._io, self, self._root) + self.blocks.append(_) + if ((self._io.is_eof()) or (_.block_type == self._root.BlockType.end_of_file)) : + break class ImageData(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 944c032d..6a27a4a8 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -87,7 +87,7 @@ class Flow(stateobject.StateObject): type=str, intercepted=bool, marked=bool, - metadata=dict, + metadata=typing.Dict[str, typing.Any], ) def get_state(self): diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index da9d2a44..51bd116b 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -1,5 +1,9 @@ """ This module handles the import of mitmproxy flows generated by old versions. + +The flow file version is decoupled from the mitmproxy release cycle (since +v3.0.0dev) and versioning. Every change or migration gets a new flow file +version number, this prevents issues with developer builds and snapshots. """ import uuid from typing import Any, Dict, Mapping, Union # noqa @@ -119,6 +123,7 @@ def convert_200_300(data): def convert_300_4(data): data["version"] = 4 + # Ths is an empty migration to transition to the new versioning scheme. return data @@ -149,6 +154,24 @@ def convert_4_5(data): return data +def convert_5_6(data): + data["version"] = 6 + data["client_conn"]["tls_established"] = data["client_conn"].pop("ssl_established") + data["client_conn"]["timestamp_tls_setup"] = data["client_conn"].pop("timestamp_ssl_setup") + data["server_conn"]["tls_established"] = data["server_conn"].pop("ssl_established") + data["server_conn"]["timestamp_tls_setup"] = data["server_conn"].pop("timestamp_ssl_setup") + if data["server_conn"]["via"]: + data["server_conn"]["via"]["tls_established"] = data["server_conn"]["via"].pop("ssl_established") + data["server_conn"]["via"]["timestamp_tls_setup"] = data["server_conn"]["via"].pop("timestamp_ssl_setup") + return data + + +def convert_6_7(data): + data["version"] = 7 + data["client_conn"]["tls_extensions"] = None + return data + + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} @@ -201,6 +224,8 @@ converters = { (2, 0): convert_200_300, (3, 0): convert_300_4, 4: convert_4_5, + 5: convert_5_6, + 6: convert_6_7, } diff --git a/mitmproxy/master.py b/mitmproxy/master.py index de3b24e1..a5e948f6 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -77,7 +77,7 @@ class Master: def add_log(self, e, level): """ - level: debug, info, warn, error + level: debug, alert, info, warn, error """ self.addons.trigger("log", log.LogEntry(e, level)) diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index d08938c9..85217794 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -301,11 +301,11 @@ class _Connection: self.rfile = None self.wfile = None - self.ssl_established = False + self.tls_established = False self.finished = False def get_current_cipher(self): - if not self.ssl_established: + if not self.tls_established: return None name = self.connection.get_cipher_name() @@ -381,7 +381,7 @@ class TCPClient(_Connection): else: close_socket(self.connection) - def convert_to_ssl(self, sni=None, alpn_protos=None, **sslctx_kwargs): + def convert_to_tls(self, sni=None, alpn_protos=None, **sslctx_kwargs): context = tls.create_client_context( alpn_protos=alpn_protos, sni=sni, @@ -400,13 +400,13 @@ class TCPClient(_Connection): else: raise exceptions.TlsException("SSL handshake error: %s" % repr(v)) - self.cert = certs.SSLCert(self.connection.get_peer_certificate()) + self.cert = certs.Cert(self.connection.get_peer_certificate()) # Keep all server certificates in a list for i in self.connection.get_peer_cert_chain(): - self.server_certs.append(certs.SSLCert(i)) + self.server_certs.append(certs.Cert(i)) - self.ssl_established = True + self.tls_established = True self.rfile.set_descriptor(self.connection) self.wfile.set_descriptor(self.connection) @@ -473,7 +473,7 @@ class TCPClient(_Connection): return self.connection.gettimeout() def get_alpn_proto_negotiated(self): - if self.ssl_established: + if self.tls_established: return self.connection.get_alpn_proto_negotiated() else: return b"" @@ -491,7 +491,7 @@ class BaseHandler(_Connection): self.server = server self.clientcert = None - def convert_to_ssl(self, cert, key, **sslctx_kwargs): + def convert_to_tls(self, cert, key, **sslctx_kwargs): """ Convert connection to SSL. For a list of parameters, see tls.create_server_context(...) @@ -507,10 +507,10 @@ class BaseHandler(_Connection): self.connection.do_handshake() except SSL.Error as v: raise exceptions.TlsException("SSL handshake error: %s" % repr(v)) - self.ssl_established = True + self.tls_established = True cert = self.connection.get_peer_certificate() if cert: - self.clientcert = certs.SSLCert(cert) + self.clientcert = certs.Cert(cert) self.rfile.set_descriptor(self.connection) self.wfile.set_descriptor(self.connection) @@ -521,7 +521,7 @@ class BaseHandler(_Connection): self.connection.settimeout(n) def get_alpn_proto_negotiated(self): - if self.ssl_established: + if self.tls_established: return self.connection.get_alpn_proto_negotiated() else: return b"" diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 74911f1e..0e43a2ac 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -2,15 +2,20 @@ # then add options to disable certain methods # https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 import binascii +import io import os +import struct import threading import typing from ssl import match_hostname, CertificateError import certifi from OpenSSL import SSL +from kaitaistruct import KaitaiStream from mitmproxy import exceptions, certs +from mitmproxy.contrib.kaitaistruct import tls_client_hello +from mitmproxy.net import check BASIC_OPTIONS = ( SSL.OP_CIPHER_SERVER_PREFERENCE @@ -189,7 +194,7 @@ def _create_ssl_context( def create_client_context( cert: str = None, sni: str = None, - address: str=None, + address: str = None, verify: int = SSL.VERIFY_NONE, **sslctx_kwargs ) -> SSL.Context: @@ -213,7 +218,7 @@ def create_client_context( ) -> bool: if is_cert_verified and depth == 0: # Verify hostname of leaf certificate. - cert = certs.SSLCert(x509) + cert = certs.Cert(x509) try: crt = dict( subjectAltName=[("DNS", x.decode("ascii", "strict")) for x in cert.altnames] @@ -270,17 +275,17 @@ def create_client_context( def create_server_context( - cert: typing.Union[certs.SSLCert, str], + cert: typing.Union[certs.Cert, str], key: SSL.PKey, handle_sni: typing.Optional[typing.Callable[[SSL.Connection], None]] = None, request_client_cert: bool = False, chain_file=None, dhparams=None, - extra_chain_certs: typing.Iterable[certs.SSLCert] = None, + extra_chain_certs: typing.Iterable[certs.Cert] = None, **sslctx_kwargs ) -> SSL.Context: """ - cert: A certs.SSLCert object or the path to a certificate + cert: A certs.Cert object or the path to a certificate chain file. handle_sni: SNI handler, should take a connection object. Server @@ -321,7 +326,7 @@ def create_server_context( ) context.use_privatekey(key) - if isinstance(cert, certs.SSLCert): + if isinstance(cert, certs.Cert): context.use_certificate(cert.x509) else: context.use_certificate_chain_file(cert) @@ -338,3 +343,119 @@ def create_server_context( SSL._lib.SSL_CTX_set_tmp_dh(context._context, dhparams) return context + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == 0x16 and + d[1] == 0x03 and + 0x0 <= d[2] <= 0x03 + ) + + +def get_client_hello(rfile): + """ + Peek into the socket and read all records that contain the initial client hello message. + + client_conn: + The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = b"" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) < 5: + raise exceptions.TlsProtocolException( + 'Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack_from("!H", record_header, 3)[0] + 5 + record_body = rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise exceptions.TlsProtocolException( + "Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4 + return client_hello + + +class ClientHello: + + def __init__(self, raw_client_hello): + self._client_hello = tls_client_hello.TlsClientHello( + KaitaiStream(io.BytesIO(raw_client_hello)) + ) + + @property + def cipher_suites(self): + return self._client_hello.cipher_suites.cipher_suites + + @property + def sni(self): + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + is_valid_sni_extension = ( + extension.type == 0x00 and + len(extension.body.server_names) == 1 and + extension.body.server_names[0].name_type == 0 and + check.is_valid_host(extension.body.server_names[0].host_name) + ) + if is_valid_sni_extension: + return extension.body.server_names[0].host_name.decode("idna") + return None + + @property + def alpn_protocols(self): + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + if extension.type == 0x10: + return list(x.name for x in extension.body.alpn_protocols) + return [] + + @property + def extensions(self) -> typing.List[typing.Tuple[int, bytes]]: + ret = [] + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + body = getattr(extension, "_raw_body", extension.body) + ret.append((extension.type, body)) + return ret + + @classmethod + def from_file(cls, client_conn) -> "ClientHello": + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + client_conn: + The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. + Returns: + :py:class:`client hello <mitmproxy.net.tls.ClientHello>`. + """ + try: + raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. + except exceptions.ProtocolException as e: + raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) + + try: + return cls(raw_client_hello) + except EOFError as e: + raise exceptions.TlsProtocolException( + 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % + (repr(e), binascii.hexlify(raw_client_hello)) + ) + + def __repr__(self): + return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \ + (self.sni, self.alpn_protocols, self.cipher_suites) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ff7edf39..862380c5 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -44,8 +44,6 @@ class Options(optmanager.OptManager): console_layout = None # type: str console_layout_headers = None # type: bool console_mouse = None # type: bool - console_order = None # type: str - console_order_reversed = None # type: bool console_palette = None # type: str console_palette_transparent = None # type: bool default_contentview = None # type: str @@ -98,6 +96,8 @@ class Options(optmanager.OptManager): upstream_cert = None # type: bool verbosity = None # type: str view_filter = None # type: Optional[str] + view_order = None # type: str + view_order_reversed = None # type: bool web_debug = None # type: bool web_iface = None # type: str web_open_browser = None # type: bool diff --git a/mitmproxy/proxy/protocol/__init__.py b/mitmproxy/proxy/protocol/__init__.py index 6dbdd13c..5860542a 100644 --- a/mitmproxy/proxy/protocol/__init__.py +++ b/mitmproxy/proxy/protocol/__init__.py @@ -36,13 +36,11 @@ from .http1 import Http1Layer from .http2 import Http2Layer from .websocket import WebSocketLayer from .rawtcp import RawTCPLayer -from .tls import TlsClientHello from .tls import TlsLayer -from .tls import is_tls_record_magic __all__ = [ "Layer", "ServerConnectionMixin", - "TlsLayer", "is_tls_record_magic", "TlsClientHello", + "TlsLayer", "UpstreamConnectLayer", "HttpLayer", "Http1Layer", diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index cc22c0b7..022e8133 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -75,7 +75,7 @@ class RequestReplayThread(basethread.BaseThread): ) if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") - server.establish_ssl( + server.establish_tls( self.options.client_certs, sni=self.f.server_conn.sni ) @@ -90,7 +90,7 @@ class RequestReplayThread(basethread.BaseThread): ) server.connect() if r.scheme == "https": - server.establish_ssl( + server.establish_tls( self.options.client_certs, sni=self.f.server_conn.sni ) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 21bf1417..d04c9801 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -1,14 +1,9 @@ -import struct from typing import Optional # noqa from typing import Union -import io -from kaitaistruct import KaitaiStream from mitmproxy import exceptions -from mitmproxy.contrib.kaitaistruct import tls_client_hello +from mitmproxy.net import tls as net_tls from mitmproxy.proxy.protocol import base -from mitmproxy.net import check - # taken from https://testssl.sh/openssl-rfc.mappping.html CIPHER_ID_NAME_MAP = { @@ -200,7 +195,6 @@ CIPHER_ID_NAME_MAP = { 0x080080: 'RC4-64-MD5', } - # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old DEFAULT_CLIENT_CIPHERS = ( @@ -216,114 +210,7 @@ DEFAULT_CLIENT_CIPHERS = ( ) -def is_tls_record_magic(d): - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes. - False, otherwise. - """ - d = d[:3] - - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return ( - len(d) == 3 and - d[0] == 0x16 and - d[1] == 0x03 and - 0x0 <= d[2] <= 0x03 - ) - - -def get_client_hello(client_conn): - """ - Peek into the socket and read all records that contain the initial client hello message. - - client_conn: - The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = b"" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise exceptions.TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise exceptions.TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4 - return client_hello - - -class TlsClientHello: - - def __init__(self, raw_client_hello): - self._client_hello = tls_client_hello.TlsClientHello(KaitaiStream(io.BytesIO(raw_client_hello))) - - def raw(self): - return self._client_hello - - @property - def cipher_suites(self): - return self._client_hello.cipher_suites.cipher_suites - - @property - def sni(self): - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - is_valid_sni_extension = ( - extension.type == 0x00 and - len(extension.body.server_names) == 1 and - extension.body.server_names[0].name_type == 0 and - check.is_valid_host(extension.body.server_names[0].host_name) - ) - if is_valid_sni_extension: - return extension.body.server_names[0].host_name.decode("idna") - return None - - @property - def alpn_protocols(self): - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - if extension.type == 0x10: - return list(x.name for x in extension.body.alpn_protocols) - return [] - - @classmethod - def from_client_conn(cls, client_conn): - """ - Peek into the connection, read the initial client hello and parse it to obtain ALPN values. - client_conn: - The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. - Returns: - :py:class:`client hello <mitmproxy.proxy.protocol.tls.TlsClientHello>`. - """ - try: - raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. - except exceptions.ProtocolException as e: - raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) - - try: - return cls(raw_client_hello) - except EOFError as e: - raise exceptions.TlsProtocolException( - 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % - (repr(e), raw_client_hello.encode("hex")) - ) - - def __repr__(self): - return "TlsClientHello( sni: %s alpn_protocols: %s, cipher_suites: %s)" % \ - (self.sni, self.alpn_protocols, self.cipher_suites) - - class TlsLayer(base.Layer): - """ The TLS layer implements transparent TLS connections. @@ -334,13 +221,13 @@ class TlsLayer(base.Layer): the server connection. """ - def __init__(self, ctx, client_tls, server_tls, custom_server_sni = None): + def __init__(self, ctx, client_tls, server_tls, custom_server_sni=None): super().__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls self._custom_server_sni = custom_server_sni - self._client_hello = None # type: Optional[TlsClientHello] + self._client_hello = None # type: Optional[net_tls.ClientHello] def __call__(self): """ @@ -355,7 +242,7 @@ class TlsLayer(base.Layer): if self._client_tls: # Peek into the connection, read the initial client hello and parse it to obtain SNI and ALPN values. try: - self._client_hello = TlsClientHello.from_client_conn(self.client_conn) + self._client_hello = net_tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") @@ -414,7 +301,7 @@ class TlsLayer(base.Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool]=None) -> None: + def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool] = None) -> None: """ Set the TLS settings for the next server connection that will be established. This function will not alter an existing connection. @@ -487,7 +374,7 @@ class TlsLayer(base.Layer): extra_certs = None try: - self.client_conn.convert_to_ssl( + self.client_conn.convert_to_tls( cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, @@ -519,12 +406,14 @@ class TlsLayer(base.Layer): # We only support http/1.1 and h2. # If the server only supports spdy (next to http/1.1), it may select that # and mitmproxy would enter TCP passthrough mode, which we want to avoid. - alpn = [x for x in self._client_hello.alpn_protocols if - not (x.startswith(b"h2-") or x.startswith(b"spdy"))] + alpn = [ + x for x in self._client_hello.alpn_protocols if + not (x.startswith(b"h2-") or x.startswith(b"spdy")) + ] if alpn and b"h2" in alpn and not self.config.options.http2: alpn.remove(b"h2") - if self.client_conn.ssl_established and self.client_conn.get_alpn_proto_negotiated(): + if self.client_conn.tls_established and self.client_conn.get_alpn_proto_negotiated(): # If the client has already negotiated an ALP, then force the # server to use the same. This can only happen if the host gets # changed after the initial connection was established. E.g.: @@ -543,7 +432,7 @@ class TlsLayer(base.Layer): ciphers_server.append(CIPHER_ID_NAME_MAP[id]) ciphers_server = ':'.join(ciphers_server) - self.server_conn.establish_ssl( + self.server_conn.establish_tls( self.config.client_certs, self.server_sni, method=self.config.openssl_method_server, diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index c0ec64c9..eb0008cf 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -1,5 +1,6 @@ from mitmproxy import log from mitmproxy import exceptions +from mitmproxy.net import tls from mitmproxy.proxy import protocol from mitmproxy.proxy import modes from mitmproxy.proxy.protocol import http @@ -45,14 +46,14 @@ class RootContext: d = top_layer.client_conn.rfile.peek(3) except exceptions.TcpException as e: raise exceptions.ProtocolException(str(e)) - client_tls = protocol.is_tls_record_magic(d) + client_tls = tls.is_tls_record_magic(d) # 1. check for --ignore if self.config.check_ignore: ignore = self.config.check_ignore(top_layer.server_conn.address) if not ignore and client_tls: try: - client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn) + client_hello = tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: @@ -76,10 +77,10 @@ class RootContext: # if the user manually sets a scheme for connect requests, we use this to decide if we # want TLS or not. if top_layer.connect_request.scheme: - tls = top_layer.connect_request.scheme == "https" + server_tls = top_layer.connect_request.scheme == "https" else: - tls = client_tls - return protocol.TlsLayer(top_layer, client_tls, tls) + server_tls = client_tls + return protocol.TlsLayer(top_layer, client_tls, server_tls) # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. if isinstance(top_layer, protocol.TlsLayer): diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 007339e8..ffaf285f 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,18 +1,12 @@ -from typing import Any -from typing import List +import typing +from typing import Any # noqa from typing import MutableMapping # noqa from mitmproxy.coretypes import serializable - - -def _is_list(cls): - # The typing module is broken on Python 3.5.0, fixed on 3.5.1. - is_list_bugfix = getattr(cls, "__origin__", False) == getattr(List[Any], "__origin__", True) - return issubclass(cls, List) or is_list_bugfix +from mitmproxy.utils import typecheck class StateObject(serializable.Serializable): - """ An object with serializable state. @@ -34,22 +28,7 @@ class StateObject(serializable.Serializable): state = {} for attr, cls in self._stateobject_attributes.items(): val = getattr(self, attr) - if val is None: - state[attr] = None - elif hasattr(val, "get_state"): - state[attr] = val.get_state() - elif _is_list(cls): - state[attr] = [x.get_state() for x in val] - elif isinstance(val, dict): - s = {} - for k, v in val.items(): - if hasattr(v, "get_state"): - s[k] = v.get_state() - else: - s[k] = v - state[attr] = s - else: - state[attr] = val + state[attr] = get_state(cls, val) return state def set_state(self, state): @@ -65,13 +44,51 @@ class StateObject(serializable.Serializable): curr = getattr(self, attr) if hasattr(curr, "set_state"): curr.set_state(val) - elif hasattr(cls, "from_state"): - obj = cls.from_state(val) - setattr(self, attr, obj) - elif _is_list(cls): - cls = cls.__parameters__[0] if cls.__parameters__ else cls.__args__[0] - setattr(self, attr, [cls.from_state(x) for x in val]) - else: # primitive types such as int, str, ... - setattr(self, attr, cls(val)) + else: + setattr(self, attr, make_object(cls, val)) if state: raise RuntimeWarning("Unexpected State in __setstate__: {}".format(state)) + + +def _process(typeinfo: typecheck.Type, val: typing.Any, make: bool) -> typing.Any: + if val is None: + return None + elif make and hasattr(typeinfo, "from_state"): + return typeinfo.from_state(val) + elif not make and hasattr(val, "get_state"): + return val.get_state() + + typename = str(typeinfo) + + if typename.startswith("typing.List"): + T = typecheck.sequence_type(typeinfo) + return [_process(T, x, make) for x in val] + elif typename.startswith("typing.Tuple"): + Ts = typecheck.tuple_types(typeinfo) + if len(Ts) != len(val): + raise ValueError("Invalid data. Expected {}, got {}.".format(Ts, val)) + return tuple( + _process(T, x, make) for T, x in zip(Ts, val) + ) + elif typename.startswith("typing.Dict"): + k_cls, v_cls = typecheck.mapping_types(typeinfo) + return { + _process(k_cls, k, make): _process(v_cls, v, make) + for k, v in val.items() + } + elif typename.startswith("typing.Any"): + # FIXME: Remove this when we remove flow.metadata + assert isinstance(val, (int, str, bool, bytes)) + return val + else: + return typeinfo(val) + + +def make_object(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: + """Create an object based on the state given in val.""" + return _process(typeinfo, val, True) + + +def get_state(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: + """Get the state of the object given as val.""" + return _process(typeinfo, val, False) diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 91747866..204c7526 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -53,8 +53,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, sec_websocket_version="13", sec_websocket_key="1234", ), - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681200, + timestamp_end=946681201, content=b'' ) resp = http.HTTPResponse( @@ -66,8 +66,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, upgrade='websocket', sec_websocket_accept=b'', ), - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681202, + timestamp_end=946681203, content=b'', ) handshake_flow = http.HTTPFlow(client_conn, server_conn) @@ -157,14 +157,15 @@ def tclient_conn(): address=("127.0.0.1", 22), clientcert=None, mitmcert=None, - ssl_established=False, - timestamp_start=1, - timestamp_ssl_setup=2, - timestamp_end=3, + tls_established=False, + timestamp_start=946681200, + timestamp_tls_setup=946681201, + timestamp_end=946681206, sni="address", cipher_name="cipher", alpn_proto_negotiated=b"http/1.1", tls_version="TLSv1.2", + tls_extensions=[(0x00, bytes.fromhex("000e00000b6578616d"))], )) c.reply = controller.DummyReply() c.rfile = io.BytesIO() @@ -182,11 +183,11 @@ def tserver_conn(): source_address=("address", 22), ip_address=("192.168.0.1", 22), cert=None, - timestamp_start=1, - timestamp_tcp_setup=2, - timestamp_ssl_setup=3, - timestamp_end=4, - ssl_established=False, + timestamp_start=946681202, + timestamp_tcp_setup=946681203, + timestamp_tls_setup=946681204, + timestamp_end=946681205, + tls_established=False, sni="address", alpn_proto_negotiated=None, tls_version="TLSv1.2", diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index cd9f3b3f..d5b52bbe 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -31,8 +31,8 @@ def treq(**kwargs): http_version=b"HTTP/1.1", headers=http.Headers(((b"header", b"qvalue"), (b"content-length", b"7"))), content=b"content", - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681200, + timestamp_end=946681201, ) default.update(kwargs) return http.Request(**default) @@ -49,8 +49,8 @@ def tresp(**kwargs): reason=b"OK", headers=http.Headers(((b"header-response", b"svalue"), (b"content-length", b"7"))), content=b"message", - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681202, + timestamp_end=946681203, ) default.update(kwargs) return http.Response(**default) diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 30e8b13b..e2088e71 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -178,5 +178,5 @@ class CommandEdit(urwid.WidgetWrap): x, y = calc_coords(self._w.get_text()[0], trans, p) return x, y - def get_value(self): + def get_edit_text(self): return self.cbuf.text diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 47a30272..8a842799 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,9 +1,10 @@ import platform +import typing +from functools import lru_cache import urwid import urwid.util -from functools import lru_cache from mitmproxy.utils import human # Detect Windows Subsystem for Linux @@ -43,41 +44,48 @@ def highlight_key(str, key, textattr="text", keyattr="key"): KEY_MAX = 30 -def format_keyvals(lst, key="key", val="text", indent=0): +def format_keyvals( + entries: typing.List[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], + key_format: str = "key", + value_format: str = "text", + indent: int = 0 +) -> typing.List[urwid.Columns]: """ - Format a list of (key, value) tuples. - - If key is None, it's treated specially: - - We assume a sub-value, and add an extra indent. - - The value is treated as a pre-formatted list of directives. + Format a list of (key, value) tuples. + + Args: + entries: The list to format. keys must be strings, values can also be None or urwid widgets. + The latter makes it possible to use the result of format_keyvals() as a value. + key_format: The display attribute for the key. + value_format: The display attribute for the value. + indent: Additional indent to apply. """ + max_key_len = max((len(k) for k, v in entries if k is not None), default=0) + max_key_len = min(max_key_len, KEY_MAX) + + if indent > 2: + indent -= 2 # We use dividechars=2 below, which already adds two empty spaces + ret = [] - if lst: - maxk = min(max(len(i[0]) for i in lst if i and i[0]), KEY_MAX) - for i, kv in enumerate(lst): - if kv is None: - ret.append(urwid.Text("")) - else: - if isinstance(kv[1], urwid.Widget): - v = kv[1] - elif kv[1] is None: - v = urwid.Text("") - else: - v = urwid.Text([(val, kv[1])]) - ret.append( - urwid.Columns( - [ - ("fixed", indent, urwid.Text("")), - ( - "fixed", - maxk, - urwid.Text([(key, kv[0] or "")]) - ), - v - ], - dividechars = 2 - ) - ) + for k, v in entries: + if v is None: + v = urwid.Text("") + elif not isinstance(v, urwid.Widget): + v = urwid.Text([(value_format, v)]) + ret.append( + urwid.Columns( + [ + ("fixed", indent, urwid.Text("")), + ( + "fixed", + max_key_len, + urwid.Text([(key_format, k)]) + ), + v + ], + dividechars=2 + ) + ) return ret @@ -205,19 +213,15 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): focus=focus, extended=extended, max_url_len=max_url_len, - - intercepted = f.intercepted, - acked = acked, - - req_timestamp = f.request.timestamp_start, - req_is_replay = f.request.is_replay, - req_method = f.request.method, - req_url = f.request.pretty_url if hostheader else f.request.url, - req_http_version = f.request.http_version, - - err_msg = f.error.msg if f.error else None, - - marked = f.marked, + intercepted=f.intercepted, + acked=acked, + req_timestamp=f.request.timestamp_start, + req_is_replay=f.request.is_replay, + req_method=f.request.method, + req_url=f.request.pretty_url if hostheader else f.request.url, + req_http_version=f.request.http_version, + err_msg=f.error.msg if f.error else None, + marked=f.marked, ) if f.response: if f.response.raw_content: @@ -232,11 +236,11 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): roundtrip = human.pretty_duration(duration) d.update(dict( - resp_code = f.response.status_code, - resp_reason = f.response.reason, - resp_is_replay = f.response.is_replay, - resp_clen = contentdesc, - roundtrip = roundtrip, + resp_code=f.response.status_code, + resp_reason=f.response.reason, + resp_is_replay=f.response.is_replay, + resp_clen=contentdesc, + roundtrip=roundtrip, )) t = f.response.headers.get("content-type") diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py index f8a3df2d..d01d9b7e 100644 --- a/mitmproxy/tools/console/defaultkeys.py +++ b/mitmproxy/tools/console/defaultkeys.py @@ -59,7 +59,7 @@ def map(km): km.add("M", "view.marked.toggle", ["flowlist"], "Toggle viewing marked flows") km.add( "n", - "console.command view.create get https://google.com", + "console.command view.create get https://example.com/", ["flowlist"], "Create a new flow" ) @@ -67,14 +67,14 @@ def map(km): "o", """ console.choose.cmd Order view.order.options - set console_order={choice} + set view_order={choice} """, ["flowlist"], "Set flow list order" ) km.add("r", "replay.client @focus", ["flowlist", "flowview"], "Replay this flow") km.add("S", "console.command replay.server ", ["flowlist"], "Start server replay") - km.add("v", "set console_order_reversed=toggle", ["flowlist"], "Reverse flow list order") + km.add("v", "set view_order_reversed=toggle", ["flowlist"], "Reverse flow list order") km.add("U", "flow.mark @all false", ["flowlist"], "Un-set all marks") km.add("w", "console.command save.file @shown ", ["flowlist"], "Save listed flows to file") km.add("V", "flow.revert @focus", ["flowlist", "flowview"], "Revert changes to this flow") diff --git a/mitmproxy/tools/console/eventlog.py b/mitmproxy/tools/console/eventlog.py index 911aeb91..8083180d 100644 --- a/mitmproxy/tools/console/eventlog.py +++ b/mitmproxy/tools/console/eventlog.py @@ -47,7 +47,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): if log.log_tier(self.master.options.verbosity) < log.log_tier(entry.level): return txt = "%s: %s" % (entry.level, str(entry.msg)) - if entry.level in ("error", "warn"): + if entry.level in ("error", "warn", "alert"): e = urwid.Text((entry.level, txt)) else: e = urwid.Text(txt) diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 28fe1fbc..443ca526 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -23,157 +23,157 @@ def flowdetails(state, flow: http.HTTPFlow): metadata = flow.metadata if metadata is not None and len(metadata) > 0: - parts = [[str(k), repr(v)] for k, v in metadata.items()] + parts = [(str(k), repr(v)) for k, v in metadata.items()] text.append(urwid.Text([("head", "Metadata:")])) - text.extend(common.format_keyvals(parts, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(parts, indent=4)) if sc is not None and sc.ip_address: text.append(urwid.Text([("head", "Server Connection:")])) parts = [ - ["Address", human.format_address(sc.address)], + ("Address", human.format_address(sc.address)), ] if sc.ip_address: - parts.append(["Resolved Address", human.format_address(sc.ip_address)]) + parts.append(("Resolved Address", human.format_address(sc.ip_address))) if resp: - parts.append(["HTTP Version", resp.http_version]) + parts.append(("HTTP Version", resp.http_version)) if sc.alpn_proto_negotiated: - parts.append(["ALPN", sc.alpn_proto_negotiated]) + parts.append(("ALPN", sc.alpn_proto_negotiated)) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) c = sc.cert if c: text.append(urwid.Text([("head", "Server Certificate:")])) parts = [ - ["Type", "%s, %s bits" % c.keyinfo], - ["SHA1 digest", c.digest("sha1")], - ["Valid to", str(c.notafter)], - ["Valid from", str(c.notbefore)], - ["Serial", str(c.serial)], - [ + ("Type", "%s, %s bits" % c.keyinfo), + ("SHA1 digest", c.digest("sha1")), + ("Valid to", str(c.notafter)), + ("Valid from", str(c.notbefore)), + ("Serial", str(c.serial)), + ( "Subject", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals( c.subject, - key="highlight", - val="text" + key_format="highlight" ) ), len(c.subject) ) - ], - [ + ), + ( "Issuer", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals( - c.issuer, key="highlight", val="text" + c.issuer, + key_format="highlight" ) ), len(c.issuer) ) - ] + ) ] if c.altnames: parts.append( - [ + ( "Alt names", ", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames) - ] + ) ) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) if cc is not None: text.append(urwid.Text([("head", "Client Connection:")])) parts = [ - ["Address", "{}:{}".format(cc.address[0], cc.address[1])], + ("Address", "{}:{}".format(cc.address[0], cc.address[1])), ] if req: - parts.append(["HTTP Version", req.http_version]) + parts.append(("HTTP Version", req.http_version)) if cc.tls_version: - parts.append(["TLS Version", cc.tls_version]) + parts.append(("TLS Version", cc.tls_version)) if cc.sni: - parts.append(["Server Name Indication", cc.sni]) + parts.append(("Server Name Indication", cc.sni)) if cc.cipher_name: - parts.append(["Cipher Name", cc.cipher_name]) + parts.append(("Cipher Name", cc.cipher_name)) if cc.alpn_proto_negotiated: - parts.append(["ALPN", cc.alpn_proto_negotiated]) + parts.append(("ALPN", cc.alpn_proto_negotiated)) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) parts = [] if cc is not None and cc.timestamp_start: parts.append( - [ + ( "Client conn. established", maybe_timestamp(cc, "timestamp_start") - ] + ) ) - if cc.ssl_established: + if cc.tls_established: parts.append( - [ + ( "Client conn. TLS handshake", - maybe_timestamp(cc, "timestamp_ssl_setup") - ] + maybe_timestamp(cc, "timestamp_tls_setup") + ) ) if sc is not None and sc.timestamp_start: parts.append( - [ + ( "Server conn. initiated", maybe_timestamp(sc, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Server conn. TCP handshake", maybe_timestamp(sc, "timestamp_tcp_setup") - ] + ) ) - if sc.ssl_established: + if sc.tls_established: parts.append( - [ + ( "Server conn. TLS handshake", - maybe_timestamp(sc, "timestamp_ssl_setup") - ] + maybe_timestamp(sc, "timestamp_tls_setup") + ) ) if req is not None and req.timestamp_start: parts.append( - [ + ( "First request byte", maybe_timestamp(req, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Request complete", maybe_timestamp(req, "timestamp_end") - ] + ) ) if resp is not None and resp.timestamp_start: parts.append( - [ + ( "First response byte", maybe_timestamp(resp, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Response complete", maybe_timestamp(resp, "timestamp_end") - ] + ) ) if parts: @@ -181,6 +181,6 @@ def flowdetails(state, flow: http.HTTPFlow): parts = sorted(parts, key=lambda p: p[1]) text.append(urwid.Text([("head", "Timing:")])) - text.extend(common.format_keyvals(parts, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(parts, indent=4)) return searchable.Searchable(text) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 05d2573f..a4b629d4 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -13,6 +13,7 @@ from mitmproxy.tools.console import flowdetailview from mitmproxy.tools.console import searchable from mitmproxy.tools.console import tabs import mitmproxy.tools.console.master # noqa +from mitmproxy.utils import strutils class SearchError(Exception): @@ -152,10 +153,31 @@ class FlowDetails(tabs.Tabs): def conn_text(self, conn): if conn: + hdrs = [] + for k, v in conn.headers.fields: + # This will always force an ascii representation of headers. For example, if the server sends a + # + # X-Authors: Made with ❤ in Hamburg + # + # header, mitmproxy will display the following: + # + # X-Authors: Made with \xe2\x9d\xa4 in Hamburg. + # + # The alternative would be to just use the header's UTF-8 representation and maybe + # do `str.replace("\t", "\\t")` to exempt tabs from urwid's special characters escaping [1]. + # That would in some terminals allow rendering UTF-8 characters, but the mapping + # wouldn't be bijective, i.e. a user couldn't distinguish "\\t" and "\t". + # Also, from a security perspective, a mitmproxy user couldn't be fooled by homoglyphs. + # + # 1) https://github.com/mitmproxy/mitmproxy/issues/1833 + # https://github.com/urwid/urwid/blob/6608ee2c9932d264abd1171468d833b7a4082e13/urwid/display_common.py#L35-L36, + + k = strutils.bytes_to_escaped_str(k) + ":" + v = strutils.bytes_to_escaped_str(v) + hdrs.append((k, v)) txt = common.format_keyvals( - [(h + ":", v) for (h, v) in conn.headers.items(multi=True)], - key = "header", - val = "text" + hdrs, + key_format="header" ) viewmode = self.master.commands.call("console.flowview.mode") msg, body = self.content_view(viewmode, conn) diff --git a/mitmproxy/tools/console/grideditor/col.py b/mitmproxy/tools/console/grideditor/col.py deleted file mode 100644 index 3331f3e7..00000000 --- a/mitmproxy/tools/console/grideditor/col.py +++ /dev/null @@ -1,67 +0,0 @@ -import typing - -import urwid - -from mitmproxy.tools.console import signals -from mitmproxy.tools.console.grideditor import base -from mitmproxy.utils import strutils - -strbytes = typing.Union[str, bytes] - - -class Column(base.Column): - def Display(self, data): - return Display(data) - - def Edit(self, data): - return Edit(data) - - def blank(self): - return "" - - def keypress(self, key, editor): - if key in ["m_select"]: - editor.walker.start_edit() - else: - return key - - -class Display(base.Cell): - def __init__(self, data: strbytes) -> None: - self.data = data - if isinstance(data, bytes): - escaped = strutils.bytes_to_escaped_str(data) - else: - escaped = data.encode() - w = urwid.Text(escaped, wrap="any") - super().__init__(w) - - def get_data(self) -> strbytes: - return self.data - - -class Edit(base.Cell): - def __init__(self, data: strbytes) -> None: - if isinstance(data, bytes): - escaped = strutils.bytes_to_escaped_str(data) - else: - escaped = data.encode() - self.type = type(data) # type: typing.Type - w = urwid.Edit(edit_text=escaped, wrap="any", multiline=True) - w = urwid.AttrWrap(w, "editfield") - super().__init__(w) - - def get_data(self) -> strbytes: - txt = self._w.get_text()[0].strip() - try: - if self.type == bytes: - return strutils.escaped_str_to_bytes(txt) - else: - return txt.decode() - except ValueError: - signals.status_message.send( - self, - message="Invalid Python-style string encoding.", - expire=1000 - ) - raise diff --git a/mitmproxy/tools/console/grideditor/col_text.py b/mitmproxy/tools/console/grideditor/col_text.py index f0ac06f8..32518670 100644 --- a/mitmproxy/tools/console/grideditor/col_text.py +++ b/mitmproxy/tools/console/grideditor/col_text.py @@ -21,7 +21,7 @@ class Column(col_bytes.Column): return TEdit(data, self.encoding_args) def blank(self): - return u"" + return "" # This is the same for both edit and display. diff --git a/mitmproxy/tools/console/grideditor/col_viewany.py b/mitmproxy/tools/console/grideditor/col_viewany.py new file mode 100644 index 00000000..f5d35eee --- /dev/null +++ b/mitmproxy/tools/console/grideditor/col_viewany.py @@ -0,0 +1,33 @@ +""" +A display-only column that displays any data type. +""" + +import typing + +import urwid +from mitmproxy.tools.console.grideditor import base +from mitmproxy.utils import strutils + + +class Column(base.Column): + def Display(self, data): + return Display(data) + + Edit = Display + + def blank(self): + return "" + + +class Display(base.Cell): + def __init__(self, data: typing.Any) -> None: + self.data = data + if isinstance(data, bytes): + data = strutils.bytes_to_escaped_str(data) + if not isinstance(data, str): + data = repr(data) + w = urwid.Text(data, wrap="any") + super().__init__(w) + + def get_data(self) -> typing.Any: + return self.data diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index b5d16737..fbe48a1a 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,13 +1,14 @@ +import typing from mitmproxy import exceptions +from mitmproxy.net.http import Headers from mitmproxy.tools.console import layoutwidget +from mitmproxy.tools.console import signals from mitmproxy.tools.console.grideditor import base -from mitmproxy.tools.console.grideditor import col -from mitmproxy.tools.console.grideditor import col_text from mitmproxy.tools.console.grideditor import col_bytes from mitmproxy.tools.console.grideditor import col_subgrid -from mitmproxy.tools.console import signals -from mitmproxy.net.http import Headers +from mitmproxy.tools.console.grideditor import col_text +from mitmproxy.tools.console.grideditor import col_viewany class QueryEditor(base.FocusEditor): @@ -67,7 +68,6 @@ class RequestFormEditor(base.FocusEditor): class PathEditor(base.FocusEditor): # TODO: Next row on enter? - title = "Edit Path Components" columns = [ col_text.Column("Component"), @@ -175,11 +175,22 @@ class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): class DataViewer(base.GridEditor, layoutwidget.LayoutWidget): title = None # type: str - def __init__(self, master, vals): + def __init__( + self, + master, + vals: typing.Union[ + typing.List[typing.List[typing.Any]], + typing.List[typing.Any], + str, + ]) -> None: if vals: + # Whatever vals is, make it a list of rows containing lists of column values. + if isinstance(vals, str): + vals = [vals] if not isinstance(vals[0], list): vals = [[i] for i in vals] - self.columns = [col.Column("")] * len(vals[0]) + + self.columns = [col_viewany.Column("")] * len(vals[0]) super().__init__(master, vals, self.callback) def callback(self, vals): diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 439289f6..1b4b9ac6 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -76,7 +76,7 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): def filtexp(self): text = [] - text.extend(common.format_keyvals(flowfilter.help, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(flowfilter.help, indent=4)) text.append( urwid.Text( [ @@ -96,7 +96,7 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] text.extend( - common.format_keyvals(examples, key="key", val="text", indent=4) + common.format_keyvals(examples, indent=4) ) return CListBox(text) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 04c7cc0c..da35047e 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -88,7 +88,7 @@ class ConsoleMaster(master.Master): def sig_add_log(self, event_store, entry: log.LogEntry): if log.log_tier(self.options.verbosity) < log.log_tier(entry.level): return - if entry.level in ("error", "warn"): + if entry.level in ("error", "warn", "alert"): if self.first_tick: self.start_err = entry else: diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 465fd574..df69ff2f 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -24,7 +24,7 @@ class Palette: # List and Connections 'method', 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', - 'error', "warn", + 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', # Hex view @@ -103,6 +103,7 @@ class LowDark(Palette): code_500 = ('light red', 'default'), code_other = ('dark red', 'default'), + alert = ('light magenta', 'default'), warn = ('brown', 'default'), error = ('light red', 'default'), @@ -176,6 +177,7 @@ class LowLight(Palette): error = ('light red', 'default'), warn = ('brown', 'default'), + alert = ('light magenta', 'default'), header = ('dark blue', 'default'), highlight = ('black,bold', 'default'), @@ -265,6 +267,7 @@ class SolarizedLight(LowLight): error = (sol_red, 'default'), warn = (sol_orange, 'default'), + alert = (sol_magenta, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), @@ -319,6 +322,7 @@ class SolarizedDark(LowDark): error = (sol_red, 'default'), warn = (sol_orange, 'default'), + alert = (sol_magenta, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 572b70fc..09cfd58a 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -101,7 +101,7 @@ class ActionBar(urwid.WidgetWrap): elif k in self.onekey: self.prompt_execute(k) elif k == "enter": - self.prompt_execute(self._w.get_value()) + self.prompt_execute(self._w.get_edit_text()) else: if common.is_keypress(k): self._w.keypress(size, k) diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 87680f6e..c7bce7d3 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -16,7 +16,10 @@ from mitmproxy.tools.console import eventlog class StackWidget(urwid.Frame): - def __init__(self, widget, title, focus): + def __init__(self, window, widget, title, focus): + self.is_focused = focus + self.window = window + if title: header = urwid.AttrWrap( urwid.Text(title), @@ -29,6 +32,11 @@ class StackWidget(urwid.Frame): header=header ) + def mouse_event(self, size, event, button, col, row, focus): + if event == "mouse press" and button == 1 and not self.is_focused: + self.window.switch() + return super().mouse_event(size, event, button, col, row, focus) + def keypress(self, size, key): # Make sure that we don't propagate cursor events outside of the widget. # Otherwise, in a horizontal layout, urwid's Pile would change the focused widget @@ -162,6 +170,7 @@ class Window(urwid.Frame): else: title = None return StackWidget( + self, widget, title, self.pane == idx @@ -234,28 +243,34 @@ class Window(urwid.Frame): self.view_changed() self.focus_changed() - def current(self, keyctx): + def stacks_sorted_by_focus(self): """ - Returns the active widget, but only the current focus or overlay has - a matching key context. + Returns: + self.stacks, with the focused stack first. """ - t = self.focus_stack().top_widget() - if t.keyctx == keyctx: - return t + stacks = self.stacks.copy() + stacks.insert(0, stacks.pop(self.pane)) + return stacks - def current_window(self, keyctx): + def current(self, keyctx): """ - Returns the active window, ignoring overlays. + Returns the active widget with a matching key context, including overlays. + If multiple stacks have an active widget with a matching key context, + the currently focused stack is preferred. """ - t = self.focus_stack().top_window() - if t.keyctx == keyctx: - return t + for s in self.stacks_sorted_by_focus(): + t = s.top_widget() + if t.keyctx == keyctx: + return t - def any(self, keyctx): + def current_window(self, keyctx): """ - Returns the top window of either stack if they match the context. + Returns the active window with a matching key context, ignoring overlays. + If multiple stacks have an active widget with a matching key context, + the currently focused stack is preferred. """ - for t in [x.top_window() for x in self.stacks]: + for s in self.stacks_sorted_by_focus(): + t = s.top_window() if t.keyctx == keyctx: return t diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 77695515..36c9d917 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -43,6 +43,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: continue f[conn]["alpn_proto_negotiated"] = \ f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace") + # There are some bytes in here as well, let's skip it until we have them in the UI. + f["client_conn"].pop("tls_extensions", None) if flow.error: f["error"] = flow.error.get_state() diff --git a/mitmproxy/types.py b/mitmproxy/types.py index 8ae8b309..3875128d 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -267,14 +267,14 @@ class _CutSpecType(_BaseType): "client_conn.address.host", "client_conn.tls_version", "client_conn.sni", - "client_conn.ssl_established", + "client_conn.tls_established", "server_conn.address.port", "server_conn.address.host", "server_conn.ip_address.host", "server_conn.tls_version", "server_conn.sni", - "server_conn.ssl_established", + "server_conn.tls_established", ] def completion(self, manager: _CommandBase, t: type, s: str) -> typing.Sequence[str]: diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index 73f7047c..873bef06 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -66,9 +66,9 @@ REPLACEMENTS = { "--palette": "console_palette", "--palette-transparent": "console_palette_transparent:", "--follow": "console_focus_follow", - "--order": "console_order", + "--order": "view_order", "--no-mouse": "console_mouse", - "--reverse": "console_order_reversed", + "--reverse": "view_order_reversed", "--no-http2-priority": "http2_priority", "--no-websocket": "websocket", "--no-upstream-cert": "upstream_cert", diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 1070fad0..22db68f5 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -1,7 +1,40 @@ import typing +Type = typing.Union[ + typing.Any # anything more elaborate really fails with mypy at the moment. +] -def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> None: + +def sequence_type(typeinfo: typing.Type[typing.List]) -> Type: + """Return the type of a sequence, e.g. typing.List""" + try: + return typeinfo.__args__[0] # type: ignore + except AttributeError: # Python 3.5.0 + return typeinfo.__parameters__[0] # type: ignore + + +def tuple_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: + """Return the types of a typing.Tuple""" + try: + return typeinfo.__args__ # type: ignore + except AttributeError: # Python 3.5.x + return typeinfo.__tuple_params__ # type: ignore + + +def union_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: + """return the types of a typing.Union""" + try: + return typeinfo.__args__ # type: ignore + except AttributeError: # Python 3.5.x + return typeinfo.__union_params__ # type: ignore + + +def mapping_types(typeinfo: typing.Type[typing.Mapping]) -> typing.Tuple[Type, Type]: + """return the types of a mapping, e.g. typing.Dict""" + return typeinfo.__args__ # type: ignore + + +def check_option_type(name: str, value: typing.Any, typeinfo: Type) -> None: """ Check if the provided value is an instance of typeinfo and raises a TypeError otherwise. This function supports only those types required for @@ -16,13 +49,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non typename = str(typeinfo) if typename.startswith("typing.Union"): - try: - types = typeinfo.__args__ # type: ignore - except AttributeError: - # Python 3.5.x - types = typeinfo.__union_params__ # type: ignore - - for T in types: + for T in union_types(typeinfo): try: check_option_type(name, value, T) except TypeError: @@ -31,12 +58,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non return raise e elif typename.startswith("typing.Tuple"): - try: - types = typeinfo.__args__ # type: ignore - except AttributeError: - # Python 3.5.x - types = typeinfo.__tuple_params__ # type: ignore - + types = tuple_types(typeinfo) if not isinstance(value, (tuple, list)): raise e if len(types) != len(value): @@ -45,11 +67,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non check_option_type("{}[{}]".format(name, i), x, T) return elif typename.startswith("typing.Sequence"): - try: - T = typeinfo.__args__[0] # type: ignore - except AttributeError: - # Python 3.5.0 - T = typeinfo.__parameters__[0] # type: ignore + T = sequence_type(typeinfo) if not isinstance(value, (tuple, list)): raise e for v in value: diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 3073c3d3..c2cb3822 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -9,7 +9,7 @@ MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one # for each change in the file format. -FLOW_FORMAT_VERSION = 5 +FLOW_FORMAT_VERSION = 7 def get_version(dev: bool = False, build: bool = False, refresh: bool = False) -> str: @@ -33,7 +33,7 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) try: git_describe = subprocess.check_output( - ['git', 'describe', '--tags', '--long'], + ['git', 'describe', '--long'], stderr=subprocess.STDOUT, cwd=here, ) @@ -48,7 +48,7 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - # Add suffix for non-tagged releases if tag_dist > 0: - mitmproxy_version += ".dev{tag_dist:04}".format(tag_dist=tag_dist) + mitmproxy_version += ".dev{tag_dist}".format(tag_dist=tag_dist) # The wheel build tag (we use the commit) must start with a digit, so we include "0x" mitmproxy_version += "-0x{commit}".format(commit=commit) @@ -60,5 +60,5 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - return mitmproxy_version -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover print(VERSION) diff --git a/pathod/pathoc.py b/pathod/pathoc.py index e5fe4c2d..b177d556 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -79,7 +79,7 @@ class SSLInfo: } t = types.get(pk.type(), "Uknown") parts.append("\tPubkey: %s bit %s" % (pk.bits(), t)) - s = certs.SSLCert(i) + s = certs.Cert(i) if s.altnames: parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames)) return "\n".join(parts) @@ -313,7 +313,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2: alpn_protos.append(b'h2') - self.convert_to_ssl( + self.convert_to_tls( sni=self.sni, cert=self.clientcert, method=self.ssl_version, diff --git a/pathod/pathod.py b/pathod/pathod.py index f8e64f9e..17db57ee 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -170,7 +170,7 @@ class PathodHandler(tcp.BaseHandler): ), cipher=None, ) - if self.ssl_established: + if self.tls_established: retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() @@ -244,7 +244,7 @@ class PathodHandler(tcp.BaseHandler): if self.server.ssl: try: cert, key, _ = self.server.ssloptions.get_cert(None) - self.convert_to_ssl( + self.convert_to_tls( cert, key, handle_sni=self.handle_sni, diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py index 4387b4fb..5fcb6618 100644 --- a/pathod/protocols/http.py +++ b/pathod/protocols/http.py @@ -27,7 +27,7 @@ class HTTPProtocol: cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert( connect[0].encode() ) - self.pathod_handler.convert_to_ssl( + self.pathod_handler.convert_to_tls( cert, key, handle_sni=self.pathod_handler.handle_sni, diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 2d1f1bf6..63e6ee0b 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -30,7 +30,7 @@ class WebsocketsProtocol: ), cipher=None, ) - if self.pathod_handler.ssl_established: + if self.pathod_handler.tls_established: retlog["cipher"] = self.pathod_handler.get_current_cipher() self.pathod_handler.addlog(retlog) ld = language.websockets.NESTED_LEADER diff --git a/release/.gitignore b/release/.gitignore index 2247d5f9..905eec6e 100644 --- a/release/.gitignore +++ b/release/.gitignore @@ -1,2 +1,3 @@ /build /dist +known_hosts diff --git a/release/README.md b/release/README.md index a60b7f98..7bb89638 100644 --- a/release/README.md +++ b/release/README.md @@ -5,6 +5,10 @@ Make sure run all these steps on the correct branch you want to create a new rel - Update CHANGELOG - Verify that all CI tests pass - Tag the release and push to Github + - For alphas, betas, and release candidates, use lightweight tags. + This is necessary so that the .devXXXX counter does not reset. + - For final releases, use annotated tags. + This makes the .devXXXX counter reset. - Wait for tag CI to complete ## GitHub Release diff --git a/release/known_hosts.enc b/release/known_hosts.enc new file mode 100644 index 00000000..585ee678 --- /dev/null +++ b/release/known_hosts.enc @@ -0,0 +1 @@ +gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7
\ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 4a07885c..9050107e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -299,11 +299,15 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, """ Upload snapshot to snapshot server """ + cnopts = pysftp.CnOpts( + knownhosts=join(RELEASE_DIR, 'known_hosts') + ) with pysftp.Connection(host=host, port=port, username=user, private_key=private_key, - private_key_pass=private_key_password) as sftp: + private_key_pass=private_key_password, + cnopts=cnopts) as sftp: dir_name = "snapshots/v{}".format(get_version()) sftp.makedirs(dir_name) with sftp.cd(dir_name): @@ -75,7 +75,6 @@ exclude = mitmproxy/proxy/protocol/tls.py mitmproxy/proxy/root_context.py mitmproxy/proxy/server.py - mitmproxy/stateobject.py mitmproxy/utils/bits.py pathod/language/actions.py pathod/language/base.py @@ -105,7 +105,7 @@ setup( ], 'examples': [ "beautifulsoup4>=4.4.1, <4.7", - "Pillow>=4.3,<4.4", + "Pillow>=4.3,<5.1", ] } ) diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 2dc7eb92..3f990668 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -52,6 +52,10 @@ class TestClientPlayback: cp.stop_replay() assert not cp.flows + df = tflow.DummyFlow(tflow.tclient_conn(), tflow.tserver_conn(), True) + with pytest.raises(exceptions.CommandError, match="Can't replay live flow."): + cp.start_replay([df]) + def test_load_file(self, tmpdir): cp = clientplayback.ClientPlayback() with taddons.context(): diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index 71e699db..c444b8ee 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -23,8 +23,8 @@ def test_extract(): ["request.text", "content"], ["request.content", b"content"], ["request.raw_content", b"content"], - ["request.timestamp_start", "1"], - ["request.timestamp_end", "2"], + ["request.timestamp_start", "946681200"], + ["request.timestamp_end", "946681201"], ["request.header[header]", "qvalue"], ["response.status_code", "200"], @@ -33,30 +33,29 @@ def test_extract(): ["response.content", b"message"], ["response.raw_content", b"message"], ["response.header[header-response]", "svalue"], - ["response.timestamp_start", "1"], - ["response.timestamp_end", "2"], + ["response.timestamp_start", "946681202"], + ["response.timestamp_end", "946681203"], ["client_conn.address.port", "22"], ["client_conn.address.host", "127.0.0.1"], ["client_conn.tls_version", "TLSv1.2"], ["client_conn.sni", "address"], - ["client_conn.ssl_established", "false"], + ["client_conn.tls_established", "false"], ["server_conn.address.port", "22"], ["server_conn.address.host", "address"], ["server_conn.ip_address.host", "192.168.0.1"], ["server_conn.tls_version", "TLSv1.2"], ["server_conn.sni", "address"], - ["server_conn.ssl_established", "false"], + ["server_conn.tls_established", "false"], ] - for t in tests: - ret = cut.extract(t[0], tf) - if ret != t[1]: - raise AssertionError("%s: Expected %s, got %s" % (t[0], t[1], ret)) + for spec, expected in tests: + ret = cut.extract(spec, tf) + assert spec and ret == expected with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c1 = certs.SSLCert.from_pem(d) + c1 = certs.Cert.from_pem(d) tf.server_conn.cert = c1 assert "CERTIFICATE" in cut.extract("server_conn.cert", tf) diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index 1d05e137..97259d1c 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -190,7 +190,7 @@ class TestProxyAuth: with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="ldap:test:test:test") - with pytest.raises(IndexError): + with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="ldap:fake_serveruid=?dc=example,dc=com:person") with pytest.raises(exceptions.OptionsError): diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 1c76eb21..a95d059d 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -41,7 +41,7 @@ def test_order_generators(): tf = tflow.tflow(resp=True) rs = view.OrderRequestStart(v) - assert rs.generate(tf) == 1 + assert rs.generate(tf) == 946681200 rm = view.OrderRequestMethod(v) assert rm.generate(tf) == tf.request.method @@ -147,6 +147,10 @@ def test_create(): assert v[0].request.url == "http://foo.com/" v.create("get", "http://foo.com") assert len(v) == 2 + with pytest.raises(exceptions.CommandError, match="Invalid URL"): + v.create("get", "http://foo.com\\") + with pytest.raises(exceptions.CommandError, match="Invalid URL"): + v.create("get", "http://") def test_orders(): diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index a77435c9..af35bab3 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -150,10 +150,10 @@ class TestResponseUtils: n = time.time() r.headers["date"] = email.utils.formatdate(n) pre = r.headers["date"] - r.refresh(1) + r.refresh(946681202) assert pre == r.headers["date"] - r.refresh(61) + r.refresh(946681262) d = email.utils.parsedate_tz(r.headers["date"]) d = email.utils.mktime_tz(d) # Weird that this is not exact... diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index e9084be4..8c012e42 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -178,7 +178,7 @@ class TestServerSSL(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com", options=SSL.OP_ALL) + c.convert_to_tls(sni="foo.com", options=SSL.OP_ALL) testval = b"echo!\n" c.wfile.write(testval) c.wfile.flush() @@ -188,7 +188,7 @@ class TestServerSSL(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): assert not c.get_current_cipher() - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") ret = c.get_current_cipher() assert ret assert "AES" in ret[0] @@ -205,7 +205,7 @@ class TestSSLv3Only(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") class TestInvalidTrustFile(tservers.ServerTestBase): @@ -213,7 +213,7 @@ class TestInvalidTrustFile(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/generate.py") @@ -231,7 +231,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): def test_mode_default_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Verification errors should be saved even if connection isn't aborted # aborted @@ -245,7 +245,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): def test_mode_none_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(verify=SSL.VERIFY_NONE) + c.convert_to_tls(verify=SSL.VERIFY_NONE) # Verification errors should be saved even if connection isn't aborted assert c.ssl_verification_error @@ -259,7 +259,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -284,7 +284,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl( + c.convert_to_tls( verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) @@ -292,7 +292,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): def test_mode_none_should_pass_without_sni(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( verify=SSL.VERIFY_NONE, ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") ) @@ -303,7 +303,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): - c.convert_to_ssl( + c.convert_to_tls( sni="mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -322,7 +322,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): def test_mode_strict_w_pemfile_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -338,7 +338,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): def test_mode_strict_w_cadir_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") @@ -372,7 +372,7 @@ class TestSSLClientCert(tservers.ServerTestBase): def test_clientcert(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( cert=tutils.test_data.path("mitmproxy/net/data/clientcert/client.pem")) assert c.rfile.readline().strip() == b"1" @@ -380,7 +380,7 @@ class TestSSLClientCert(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl(cert=tutils.test_data.path("mitmproxy/net/data/clientcert/make")) + c.convert_to_tls(cert=tutils.test_data.path("mitmproxy/net/data/clientcert/make")) class TestSNI(tservers.ServerTestBase): @@ -400,15 +400,15 @@ class TestSNI(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") assert c.sni == "foo.com" assert c.rfile.readline() == b"foo.com" def test_idn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="mitmproxyäöüß.example.com") - assert c.ssl_established + c.convert_to_tls(sni="mitmproxyäöüß.example.com") + assert c.tls_established assert "doesn't match" not in str(c.ssl_verification_error) @@ -421,7 +421,7 @@ class TestServerCipherList(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") expected = b"['AES256-GCM-SHA384']" assert c.rfile.read(len(expected) + 2) == expected @@ -442,7 +442,7 @@ class TestServerCurrentCipher(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") assert b'AES256-GCM-SHA384' in c.rfile.readline() @@ -456,7 +456,7 @@ class TestServerCipherListError(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(Exception, match="handshake error"): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") class TestClientCipherListError(tservers.ServerTestBase): @@ -469,7 +469,7 @@ class TestClientCipherListError(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(Exception, match="cipher specification"): - c.convert_to_ssl(sni="foo.com", cipher_list="bogus") + c.convert_to_tls(sni="foo.com", cipher_list="bogus") class TestSSLDisconnect(tservers.ServerTestBase): @@ -484,7 +484,7 @@ class TestSSLDisconnect(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Excercise SSL.ZeroReturnError c.rfile.read(10) c.close() @@ -501,7 +501,7 @@ class TestSSLHardDisconnect(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Exercise SSL.SysCallError c.rfile.read(10) c.close() @@ -565,7 +565,7 @@ class TestALPNClient(tservers.ServerTestBase): def test_alpn(self, monkeypatch, alpn_protos, expected_negotiated, expected_response): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=alpn_protos) + c.convert_to_tls(alpn_protos=alpn_protos) assert c.get_alpn_proto_negotiated() == expected_negotiated assert c.rfile.readline().strip() == expected_response @@ -587,7 +587,7 @@ class TestSSLTimeOut(tservers.ServerTestBase): def test_timeout_client(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() c.settimeout(0.1) with pytest.raises(exceptions.TcpTimeout): c.rfile.read(10) @@ -605,7 +605,7 @@ class TestDHParams(tservers.ServerTestBase): def test_dhparams(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() ret = c.get_current_cipher() assert ret[0] == "DHE-RSA-AES256-SHA" @@ -801,5 +801,5 @@ class TestPeekSSL(TestPeek): def _connect(self, c): with c.connect() as conn: - c.convert_to_ssl() + c.convert_to_tls() return conn.pop() diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index d0583d34..489bf89f 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -1,3 +1,5 @@ +import io + import pytest from mitmproxy import exceptions @@ -6,6 +8,17 @@ from mitmproxy.net.tcp import TCPClient from test.mitmproxy.net.test_tcp import EchoHandler from . import tservers +CLIENT_HELLO_NO_EXTENSIONS = bytes.fromhex( + "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" + "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" + "61006200640100" +) +FULL_CLIENT_HELLO_NO_EXTENSIONS = ( + b"\x16\x03\x03\x00\x65" # record layer + b"\x01\x00\x00\x61" + # handshake header + CLIENT_HELLO_NO_EXTENSIONS +) + class TestMasterSecretLogger(tservers.ServerTestBase): handler = EchoHandler @@ -22,7 +35,7 @@ class TestMasterSecretLogger(tservers.ServerTestBase): c = TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() c.wfile.write(testval) c.wfile.flush() assert c.rfile.readline() == testval @@ -53,3 +66,92 @@ class TestTLSInvalid: with pytest.raises(exceptions.TlsException, match="ALPN error"): tls.create_client_context(alpn_select="foo", alpn_select_callback="bar") + + +def test_is_record_magic(): + assert not tls.is_tls_record_magic(b"POST /") + assert not tls.is_tls_record_magic(b"\x16\x03") + assert not tls.is_tls_record_magic(b"\x16\x03\x04") + assert tls.is_tls_record_magic(b"\x16\x03\x00") + assert tls.is_tls_record_magic(b"\x16\x03\x01") + assert tls.is_tls_record_magic(b"\x16\x03\x02") + assert tls.is_tls_record_magic(b"\x16\x03\x03") + + +def test_get_client_hello(): + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS + )) + assert tls.get_client_hello(rfile) + + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS[:30] + )) + with pytest.raises(exceptions.TlsProtocolException, message="Unexpected EOF"): + tls.get_client_hello(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"GET /" + )) + with pytest.raises(exceptions.TlsProtocolException, message="Expected TLS record"): + tls.get_client_hello(rfile) + + +class TestClientHello: + def test_no_extensions(self): + c = tls.ClientHello(CLIENT_HELLO_NO_EXTENSIONS) + assert repr(c) + assert c.sni is None + assert c.cipher_suites == [53, 47, 10, 5, 4, 9, 3, 6, 8, 96, 97, 98, 100] + assert c.alpn_protocols == [] + assert c.extensions == [] + + def test_extensions(self): + data = bytes.fromhex( + "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" + "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" + "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" + "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" + "170018" + ) + c = tls.ClientHello(data) + assert repr(c) + assert c.sni == 'example.com' + assert c.cipher_suites == [ + 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, + 49171, 49162, 49172, 156, 157, 47, 53, 10 + ] + assert c.alpn_protocols == [b'h2', b'http/1.1'] + assert c.extensions == [ + (65281, b'\x00'), + (0, b'\x00\x0e\x00\x00\x0bexample.com'), + (23, b''), + (35, b''), + (13, b'\x00\x10\x06\x01\x06\x03\x05\x01\x05\x03\x04\x01\x04\x03\x02\x01\x02\x03'), + (5, b'\x01\x00\x00\x00\x00'), + (18, b''), + (16, b'\x00\x0c\x02h2\x08http/1.1'), + (30032, b''), + (11, b'\x01\x00'), + (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18') + ] + + def test_from_file(self): + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS + )) + assert tls.ClientHello.from_file(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"" + )) + with pytest.raises(exceptions.TlsProtocolException): + tls.ClientHello.from_file(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"\x16\x03\x03\x00\x07" # record layer + b"\x01\x00\x00\x03" + # handshake header + b"foo" + )) + with pytest.raises(exceptions.TlsProtocolException, message='Cannot parse Client Hello'): + tls.ClientHello.from_file(rfile) diff --git a/test/mitmproxy/net/tools/getcertnames b/test/mitmproxy/net/tools/getcertnames index d64e5ff5..9349415f 100644 --- a/test/mitmproxy/net/tools/getcertnames +++ b/test/mitmproxy/net/tools/getcertnames @@ -7,7 +7,7 @@ from mitmproxy.net import tcp def get_remote_cert(host, port, sni): c = tcp.TCPClient((host, port)) c.connect() - c.convert_to_ssl(sni=sni) + c.convert_to_tls(sni=sni) return c.cert if len(sys.argv) > 2: diff --git a/test/mitmproxy/net/tservers.py b/test/mitmproxy/net/tservers.py index 44701aa5..22e195e3 100644 --- a/test/mitmproxy/net/tservers.py +++ b/test/mitmproxy/net/tservers.py @@ -60,7 +60,7 @@ class _TServer(tcp.TCPServer): else: method = OpenSSL.SSL.SSLv23_METHOD options = None - h.convert_to_ssl( + h.convert_to_tls( cert, key, method=method, diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 4f161ef5..194a57c9 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -141,7 +141,7 @@ class _Http2TestBase: while self.client.rfile.readline() != b"\r\n": pass - self.client.convert_to_ssl(alpn_protos=[b'h2']) + self.client.convert_to_tls(alpn_protos=[b'h2']) config = h2.config.H2Configuration( client_side=True, diff --git a/test/mitmproxy/proxy/protocol/test_tls.py b/test/mitmproxy/proxy/protocol/test_tls.py index e17ee46f..e69de29b 100644 --- a/test/mitmproxy/proxy/protocol/test_tls.py +++ b/test/mitmproxy/proxy/protocol/test_tls.py @@ -1,26 +0,0 @@ -from mitmproxy.proxy.protocol.tls import TlsClientHello - - -class TestClientHello: - - def test_no_extensions(self): - data = bytes.fromhex( - "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" - "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" - "61006200640100" - ) - c = TlsClientHello(data) - assert c.sni is None - assert c.alpn_protocols == [] - - def test_extensions(self): - data = bytes.fromhex( - "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" - "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" - "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" - "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" - "170018" - ) - c = TlsClientHello(data) - assert c.sni == 'example.com' - assert c.alpn_protocols == [b'h2', b'http/1.1'] diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index d9389faf..5cd9601c 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -101,8 +101,8 @@ class _WebSocketTestBase: response = http.http1.read_response(self.client.rfile, request) if self.ssl: - self.client.convert_to_ssl() - assert self.client.ssl_established + self.client.convert_to_tls() + assert self.client.tls_established request = http.Request( "relative", diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 8dce9bcd..8ec83d18 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -143,9 +143,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = certs.SSLCert(i.sslinfo.certchain[0]) - i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) - n_cert = certs.SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) assert i_cert == i2_cert assert i_cert != n_cert @@ -188,9 +188,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = certs.SSLCert(i.sslinfo.certchain[0]) - i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) - n_cert = certs.SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) assert i_cert == i2_cert assert i_cert != n_cert @@ -579,7 +579,7 @@ class TestSocks5SSL(tservers.SocksModeTest): p = self.pathoc_raw() with p.connect(): p.socks_connect(("localhost", self.server.port)) - p.convert_to_ssl() + p.convert_to_tls() f = p.request("get:/p/200") assert f.status_code == 200 @@ -709,7 +709,7 @@ class TestProxy(tservers.HTTPProxyTest): first_flow = self.master.state.flows[0] second_flow = self.master.state.flows[1] assert first_flow.server_conn.timestamp_tcp_setup - assert first_flow.server_conn.timestamp_ssl_setup is None + assert first_flow.server_conn.timestamp_tls_setup is None assert second_flow.server_conn.timestamp_tcp_setup assert first_flow.server_conn.timestamp_tcp_setup == second_flow.server_conn.timestamp_tcp_setup @@ -723,12 +723,13 @@ class TestProxy(tservers.HTTPProxyTest): class TestProxySSL(tservers.HTTPProxyTest): ssl = True - def test_request_ssl_setup_timestamp_presence(self): + def test_request_tls_attribute_presence(self): # tests that the ssl timestamp is present when ssl is used f = self.pathod("304:b@10k") assert f.status_code == 304 first_flow = self.master.state.flows[0] - assert first_flow.server_conn.timestamp_ssl_setup + assert first_flow.server_conn.timestamp_tls_setup + assert first_flow.client_conn.tls_extensions def test_via(self): # tests that the ssl timestamp is present when ssl is used @@ -1149,7 +1150,7 @@ class AddUpstreamCertsToClientChainMixin: def test_add_upstream_certs_to_client_chain(self): with open(self.servercert, "rb") as f: d = f.read() - upstreamCert = certs.SSLCert.from_pem(d) + upstreamCert = certs.Cert.from_pem(d) p = self.pathoc() with p.connect(): upstream_cert_found_in_client_chain = False diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 693bebc6..dcc185c0 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -136,18 +136,18 @@ class TestDummyCert: assert r.altnames == [] -class TestSSLCert: +class TestCert: def test_simple(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c1 = certs.SSLCert.from_pem(d) + c1 = certs.Cert.from_pem(d) assert c1.cn == b"google.com" assert len(c1.altnames) == 436 with open(tutils.test_data.path("mitmproxy/net/data/text_cert_2"), "rb") as f: d = f.read() - c2 = certs.SSLCert.from_pem(d) + c2 = certs.Cert.from_pem(d) assert c2.cn == b"www.inode.co.nz" assert len(c2.altnames) == 2 assert c2.digest("sha1") @@ -165,20 +165,20 @@ class TestSSLCert: def test_err_broken_sans(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert_weird1"), "rb") as f: d = f.read() - c = certs.SSLCert.from_pem(d) + c = certs.Cert.from_pem(d) # This breaks unless we ignore a decoding error. assert c.altnames is not None def test_der(self): with open(tutils.test_data.path("mitmproxy/net/data/dercert"), "rb") as f: d = f.read() - s = certs.SSLCert.from_der(d) + s = certs.Cert.from_der(d) assert s.cn def test_state(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c = certs.SSLCert.from_pem(d) + c = certs.Cert.from_pem(d) c.get_state() c2 = c.copy() @@ -188,6 +188,6 @@ class TestSSLCert: assert c == c2 assert c is not c2 - x = certs.SSLCert('') + x = certs.Cert('') x.set_state(a) assert x == c diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 83f0bd34..9e5d89f1 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -41,10 +41,10 @@ class TestClientConnection: def test_tls_established_property(self): c = tflow.tclient_conn() c.tls_established = True - assert c.ssl_established + assert c.tls_established assert c.tls_established c.tls_established = False - assert not c.ssl_established + assert not c.tls_established assert not c.tls_established def test_make_dummy(self): @@ -113,10 +113,10 @@ class TestServerConnection: def test_tls_established_property(self): c = tflow.tserver_conn() c.tls_established = True - assert c.ssl_established + assert c.tls_established assert c.tls_established c.tls_established = False - assert not c.ssl_established + assert not c.tls_established assert not c.tls_established def test_make_dummy(self): @@ -155,7 +155,7 @@ class TestServerConnection: def test_sni(self): c = connections.ServerConnection(('', 1234)) with pytest.raises(ValueError, matches='sni must be str, not '): - c.establish_ssl(None, b'foobar') + c.establish_tls(None, b'foobar') def test_state(self): c = tflow.tserver_conn() @@ -206,7 +206,7 @@ class TestClientConnectionTLS: key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, raw_key) - c.convert_to_ssl(cert, key) + c.convert_to_tls(cert, key) assert c.connected() assert c.sni == sni assert c.tls_established @@ -230,7 +230,7 @@ class TestServerConnectionTLS(tservers.ServerTestBase): def test_tls(self, clientcert): c = connections.ServerConnection(("127.0.0.1", self.port)) c.connect() - c.establish_ssl(clientcert, "foo.com") + c.establish_tls(clientcert, "foo.com") assert c.connected() assert c.sni == "foo.com" assert c.tls_established diff --git a/test/mitmproxy/test_stateobject.py b/test/mitmproxy/test_stateobject.py index d8c7a8e9..bd5d1792 100644 --- a/test/mitmproxy/test_stateobject.py +++ b/test/mitmproxy/test_stateobject.py @@ -1,101 +1,146 @@ -from typing import List +import typing + import pytest from mitmproxy.stateobject import StateObject -class Child(StateObject): +class TObject(StateObject): def __init__(self, x): self.x = x - _stateobject_attributes = dict( - x=int - ) - @classmethod def from_state(cls, state): obj = cls(None) obj.set_state(state) return obj + +class Child(TObject): + _stateobject_attributes = dict( + x=int + ) + def __eq__(self, other): return isinstance(other, Child) and self.x == other.x -class Container(StateObject): - def __init__(self): - self.child = None - self.children = None - self.dictionary = None +class TTuple(TObject): + _stateobject_attributes = dict( + x=typing.Tuple[int, Child] + ) + + +class TList(TObject): + _stateobject_attributes = dict( + x=typing.List[Child] + ) + +class TDict(TObject): _stateobject_attributes = dict( - child=Child, - children=List[Child], - dictionary=dict, + x=typing.Dict[str, Child] ) - @classmethod - def from_state(cls, state): - obj = cls() - obj.set_state(state) - return obj + +class TAny(TObject): + _stateobject_attributes = dict( + x=typing.Any + ) + + +class TSerializableChild(TObject): + _stateobject_attributes = dict( + x=Child + ) def test_simple(): a = Child(42) + assert a.get_state() == {"x": 42} b = a.copy() - assert b.get_state() == {"x": 42} a.set_state({"x": 44}) assert a.x == 44 assert b.x == 42 -def test_container(): - a = Container() - a.child = Child(42) +def test_serializable_child(): + child = Child(42) + a = TSerializableChild(child) + assert a.get_state() == { + "x": {"x": 42} + } + a.set_state({ + "x": {"x": 43} + }) + assert a.x.x == 43 + assert a.x is child b = a.copy() - assert a.child.x == b.child.x - b.child.x = 44 - assert a.child.x != b.child.x + assert a.x == b.x + assert a.x is not b.x -def test_container_list(): - a = Container() - a.children = [Child(42), Child(44)] +def test_tuple(): + a = TTuple((42, Child(43))) assert a.get_state() == { - "child": None, - "children": [{"x": 42}, {"x": 44}], - "dictionary": None, + "x": (42, {"x": 43}) } - copy = a.copy() - assert len(copy.children) == 2 - assert copy.children is not a.children - assert copy.children[0] is not a.children[0] - assert Container.from_state(a.get_state()) + b = a.copy() + a.set_state({"x": (44, {"x": 45})}) + assert a.x == (44, Child(45)) + assert b.x == (42, Child(43)) + +def test_tuple_err(): + a = TTuple(None) + with pytest.raises(ValueError, msg="Invalid data"): + a.set_state({"x": (42,)}) -def test_container_dict(): - a = Container() - a.dictionary = dict() - a.dictionary['foo'] = 'bar' - a.dictionary['bar'] = Child(44) + +def test_list(): + a = TList([Child(1), Child(2)]) assert a.get_state() == { - "child": None, - "children": None, - "dictionary": {'bar': {'x': 44}, 'foo': 'bar'}, + "x": [{"x": 1}, {"x": 2}], } copy = a.copy() - assert len(copy.dictionary) == 2 - assert copy.dictionary is not a.dictionary - assert copy.dictionary['bar'] is not a.dictionary['bar'] + assert len(copy.x) == 2 + assert copy.x is not a.x + assert copy.x[0] is not a.x[0] + + +def test_dict(): + a = TDict({"foo": Child(42)}) + assert a.get_state() == { + "x": {"foo": {"x": 42}} + } + b = a.copy() + assert list(a.x.items()) == list(b.x.items()) + assert a.x is not b.x + assert a.x["foo"] is not b.x["foo"] + + +def test_any(): + a = TAny(42) + b = a.copy() + assert a.x == b.x + + a = TAny(object()) + with pytest.raises(AssertionError): + a.get_state() def test_too_much_state(): - a = Container() - a.child = Child(42) + a = Child(42) s = a.get_state() s['foo'] = 'bar' - b = Container() with pytest.raises(RuntimeWarning): - b.set_state(s) + a.set_state(s) + + +def test_none(): + a = Child(None) + assert a.get_state() == {"x": None} + a = Child(42) + a.set_state({"x": None}) + assert a.x is None diff --git a/test/mitmproxy/test_version.py b/test/mitmproxy/test_version.py index f8d646dc..8c176542 100644 --- a/test/mitmproxy/test_version.py +++ b/test/mitmproxy/test_version.py @@ -1,3 +1,4 @@ +import pathlib import runpy import subprocess from unittest import mock @@ -6,7 +7,9 @@ from mitmproxy import version def test_version(capsys): - runpy.run_module('mitmproxy.version', run_name='__main__') + here = pathlib.Path(__file__).absolute().parent + version_file = here / ".." / ".." / "mitmproxy" / "version.py" + runpy.run_path(str(version_file), run_name='__main__') stdout, stderr = capsys.readouterr() assert len(stdout) > 0 assert stdout.strip() == version.VERSION @@ -27,7 +30,7 @@ def test_get_version(): assert version.get_version(True, True) == "3.0.0" m.return_value = b"tag-2-cafecafe" - assert version.get_version(True, True) == "3.0.0.dev0002-0xcafecaf" + assert version.get_version(True, True) == "3.0.0.dev2-0xcafecaf" - m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long') + m.side_effect = subprocess.CalledProcessError(-1, 'git describe --long') assert version.get_version(True, True) == "3.0.0" diff --git a/test/mitmproxy/tools/console/test_common.py b/test/mitmproxy/tools/console/test_common.py index 3ab4fd67..72438c49 100644 --- a/test/mitmproxy/tools/console/test_common.py +++ b/test/mitmproxy/tools/console/test_common.py @@ -1,12 +1,34 @@ +import urwid + from mitmproxy.test import tflow from mitmproxy.tools.console import common -from ....conftest import skip_appveyor - -@skip_appveyor def test_format_flow(): f = tflow.tflow(resp=True) assert common.format_flow(f, True) assert common.format_flow(f, True, hostheader=True) assert common.format_flow(f, True, extended=True) + + +def test_format_keyvals(): + assert common.format_keyvals( + [ + ("aa", "bb"), + ("cc", "dd"), + ("ee", None), + ] + ) + wrapped = urwid.BoxAdapter( + urwid.ListBox( + urwid.SimpleFocusListWalker( + common.format_keyvals([("foo", "bar")]) + ) + ), 1 + ) + assert wrapped.render((30, )) + assert common.format_keyvals( + [ + ("aa", wrapped) + ] + ) diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 3aa0dc54..9779a482 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -4,22 +4,9 @@ from mitmproxy import options from mitmproxy.test import tflow from mitmproxy.test import tutils from mitmproxy.tools import console -from mitmproxy.tools.console import common from ... import tservers -def test_format_keyvals(): - assert common.format_keyvals( - [ - ("aa", "bb"), - None, - ("cc", "dd"), - (None, "dd"), - (None, "dd"), - ] - ) - - def test_options(): assert options.Options(replay_kill_extra=True) diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 5295fff5..9cb4334e 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -93,3 +93,8 @@ def test_typesec_to_str(): assert(typecheck.typespec_to_str(typing.Optional[str])) == "optional str" with pytest.raises(NotImplementedError): typecheck.typespec_to_str(dict) + + +def test_mapping_types(): + # this is not covered by check_option_type, but still belongs in this module + assert (str, int) == typecheck.mapping_types(typing.Mapping[str, int]) diff --git a/test/pathod/protocols/test_http2.py b/test/pathod/protocols/test_http2.py index b1eebc73..95965cee 100644 --- a/test/pathod/protocols/test_http2.py +++ b/test/pathod/protocols/test_http2.py @@ -75,7 +75,7 @@ class TestCheckALPNMatch(net_tservers.ServerTestBase): def test_check_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=[b'h2']) + c.convert_to_tls(alpn_protos=[b'h2']) protocol = HTTP2StateProtocol(c) assert protocol.check_alpn() @@ -89,7 +89,7 @@ class TestCheckALPNMismatch(net_tservers.ServerTestBase): def test_check_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=[b'h2']) + c.convert_to_tls(alpn_protos=[b'h2']) protocol = HTTP2StateProtocol(c) with pytest.raises(NotImplementedError): protocol.check_alpn() @@ -207,7 +207,7 @@ class TestApplySettings(net_tservers.ServerTestBase): def test_apply_settings(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol._apply_settings({ @@ -302,7 +302,7 @@ class TestReadRequest(net_tservers.ServerTestBase): def test_read_request(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -328,7 +328,7 @@ class TestReadRequestRelative(net_tservers.ServerTestBase): def test_asterisk_form(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -351,7 +351,7 @@ class TestReadRequestAbsolute(net_tservers.ServerTestBase): def test_absolute_form(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -378,7 +378,7 @@ class TestReadResponse(net_tservers.ServerTestBase): def test_read_response(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol.connection_preface_performed = True @@ -404,7 +404,7 @@ class TestReadEmptyResponse(net_tservers.ServerTestBase): def test_read_empty_response(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol.connection_preface_performed = True diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 4b50e2a7..297b54d4 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -238,11 +238,11 @@ class TestDaemonHTTP2(PathocTestDaemon): http2_skip_connection_preface=True, ) - tmp_convert_to_ssl = c.convert_to_ssl - c.convert_to_ssl = Mock() - c.convert_to_ssl.side_effect = tmp_convert_to_ssl + tmp_convert_to_tls = c.convert_to_tls + c.convert_to_tls = Mock() + c.convert_to_tls.side_effect = tmp_convert_to_tls with c.connect(): - _, kwargs = c.convert_to_ssl.call_args + _, kwargs = c.convert_to_tls.call_args assert set(kwargs['alpn_protos']) == set([b'http/1.1', b'h2']) def test_request(self): diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index c0011952..d6522cb6 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -153,7 +153,7 @@ class CommonTests(tservers.DaemonTests): c = tcp.TCPClient(("localhost", self.d.port)) with c.connect(): if self.ssl: - c.convert_to_ssl() + c.convert_to_tls() c.wfile.write(b"foo\n\n\n") c.wfile.flush() l = self.d.last_log() @@ -241,7 +241,7 @@ class TestDaemonSSL(CommonTests): with c.connect(): c.wfile.write(b"\0\0\0\0") with pytest.raises(exceptions.TlsException): - c.convert_to_ssl() + c.convert_to_tls() l = self.d.last_log() assert l["type"] == "error" assert "SSL" in l["msg"] @@ -56,7 +56,7 @@ deps = -rrequirements.txt pyinstaller==3.3.1 twine==1.9.1 - pysftp==0.2.8 + pysftp==0.2.9 commands = mitmdump --version diff --git a/web/src/js/filt/filt.js b/web/src/js/filt/filt.js index 26058649..19a41af2 100644 --- a/web/src/js/filt/filt.js +++ b/web/src/js/filt/filt.js @@ -1929,7 +1929,7 @@ module.exports = (function() { function body(regex){ regex = new RegExp(regex, "i"); function bodyFilter(flow){ - return True; + return true; } bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return bodyFilter; @@ -1937,7 +1937,7 @@ module.exports = (function() { function requestBody(regex){ regex = new RegExp(regex, "i"); function requestBodyFilter(flow){ - return True; + return true; } requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return requestBodyFilter; @@ -1945,7 +1945,7 @@ module.exports = (function() { function responseBody(regex){ regex = new RegExp(regex, "i"); function responseBodyFilter(flow){ - return True; + return true; } responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return responseBodyFilter; @@ -2104,4 +2104,4 @@ module.exports = (function() { SyntaxError: peg$SyntaxError, parse: peg$parse }; -})();
\ No newline at end of file +})(); diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index 12959474..e4b151ad 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -1,4 +1,4 @@ -// PEG.js filter rules - see http://pegjs.majda.cz/online +// PEG.js filter rules - see https://pegjs.org/ { var flowutils = require("../flow/utils.js"); @@ -72,7 +72,7 @@ function responseCode(code){ function body(regex){ regex = new RegExp(regex, "i"); function bodyFilter(flow){ - return True; + return true; } bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return bodyFilter; @@ -80,7 +80,7 @@ function body(regex){ function requestBody(regex){ regex = new RegExp(regex, "i"); function requestBodyFilter(flow){ - return True; + return true; } requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return requestBodyFilter; @@ -88,7 +88,7 @@ function requestBody(regex){ function responseBody(regex){ regex = new RegExp(regex, "i"); function responseBodyFilter(flow){ - return True; + return true; } responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return responseBodyFilter; |
