From ba674ad5514c5f30315fc688a07fdac634d94dfc Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 1 Mar 2013 09:05:39 +1300 Subject: New SNI handling mechanism. --- libmproxy/proxy.py | 57 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 7c229064..c9ceb8de 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -80,8 +80,7 @@ class ServerConnection(tcp.TCPClient): def terminate(self): try: - if not self.wfile.closed: - self.wfile.flush() + self.wfile.flush() self.connection.close() except IOError: pass @@ -110,6 +109,27 @@ class RequestReplayThread(threading.Thread): self.channel.ask(err) +class HandleSNI: + def __init__(self, handler, client_conn, host, port, cert, key): + self.handler, self.client_conn, self.host, self.port = handler, client_conn, host, port + self.cert, self.key = cert, key + + def __call__(self, connection): + try: + sn = connection.get_servername() + if sn: + self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn) + new_context = SSL.Context(SSL.TLSv1_METHOD) + new_context.use_privatekey_file(self.key) + new_context.use_certificate_file(self.cert) + connection.set_context(new_context) + self.handler.sni = sn.decode("utf8").encode("idna") + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except Exception, e: + pass + + class ProxyHandler(tcp.BaseHandler): def __init__(self, config, connection, client_address, server, channel, server_version): self.channel, self.server_version = channel, server_version @@ -266,18 +286,15 @@ class ProxyHandler(tcp.BaseHandler): l = Log(msg) self.channel.tell(l) - def find_cert(self, host, port, sni): + def find_cert(self, cc, host, port, sni): if self.config.certfile: return self.config.certfile else: sans = [] if not self.config.no_upstream_cert: - try: - cert = certutils.get_remote_cert(host, port, sni) - except tcp.NetLibError, v: - raise ProxyError(502, "Unable to get remote cert: %s"%str(v)) - sans = cert.altnames - host = cert.cn.decode("utf8").encode("idna") + conn = self.get_server_connection(cc, "https", host, port, sni) + sans = conn.cert.altnames + host = conn.cert.cn.decode("utf8").encode("idna") ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") @@ -292,11 +309,6 @@ class ProxyHandler(tcp.BaseHandler): line = fp.readline() return line - def handle_sni(self, conn): - sn = conn.get_servername() - if sn: - self.sni = sn.decode("utf8").encode("idna") - def read_request_transparent(self, client_conn): orig = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not orig: @@ -304,9 +316,13 @@ class ProxyHandler(tcp.BaseHandler): host, port = orig if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" - certfile = self.find_cert(host, port, None) + dummycert = self.find_cert(client_conn, host, port, host) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) else: @@ -346,9 +362,14 @@ class ProxyHandler(tcp.BaseHandler): '\r\n' ) self.wfile.flush() - certfile = self.find_cert(host, port, None) + certfile = self.find_cert(client_conn, host, port, host) + + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) + self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) self.proxy_connect_state = (host, port, httpversion) -- cgit v1.2.3 From 10db82e9a030235ab884e70d1809ad6d673c2d13 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 14:52:05 +1300 Subject: Test SNI for ordinary proxy connections. --- libmproxy/proxy.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index c9ceb8de..54cb6f8e 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -126,7 +126,7 @@ class HandleSNI: self.handler.sni = sn.decode("utf8").encode("idna") # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. - except Exception, e: + except Exception, e: # pragma: no cover pass @@ -141,6 +141,8 @@ class ProxyHandler(tcp.BaseHandler): def get_server_connection(self, cc, scheme, host, port, sni): sc = self.server_conn + if not sni: + sni = host if sc and (scheme, host, port, sni) != (sc.scheme, sc.host, sc.port, sc.sni): sc.terminate() self.server_conn = None @@ -214,7 +216,7 @@ class ProxyHandler(tcp.BaseHandler): # the case, we want to reconnect without sending an error # to the client. while 1: - sc = self.get_server_connection(cc, scheme, host, port, host) + sc = self.get_server_connection(cc, scheme, host, port, self.sni) sc.send(request) sc.rfile.reset_timestamps() try: @@ -362,14 +364,13 @@ class ProxyHandler(tcp.BaseHandler): '\r\n' ) self.wfile.flush() - certfile = self.find_cert(client_conn, host, port, host) - - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) + dummycert = self.find_cert(client_conn, host, port, host) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert, handle_sni=sni) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) self.proxy_connect_state = (host, port, httpversion) -- cgit v1.2.3 From a95d78438c7197b0b6643a61899914083de70da9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 15:06:49 +1300 Subject: Test SNI for transparent mode. --- libmproxy/proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 54cb6f8e..964c15a9 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -140,6 +140,13 @@ class ProxyHandler(tcp.BaseHandler): tcp.BaseHandler.__init__(self, connection, client_address, server) def get_server_connection(self, cc, scheme, host, port, sni): + """ + When SNI is in play, this means we have an SSL-encrypted + connection, which means that the entire handler is dedicated to a + single server connection - no multiplexing. If this assumption ever + breaks, we'll have to do something different with the SNI host + variable on the handler object. + """ sc = self.server_conn if not sni: sni = host @@ -329,7 +336,6 @@ class ProxyHandler(tcp.BaseHandler): raise ProxyError(400, str(v)) else: scheme = "http" - host = self.sni or host line = self.get_line(self.rfile) if line == "": return None -- cgit v1.2.3 From 415844511c19b17743b42a5833590d1d683427d2 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 16:59:16 +1300 Subject: Test cert generation errors. --- libmproxy/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 964c15a9..a6a72d55 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -306,7 +306,7 @@ class ProxyHandler(tcp.BaseHandler): host = conn.cert.cn.decode("utf8").encode("idna") ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: - raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") + raise ProxyError(502, "Unable to generate dummy cert.") return ret def get_line(self, fp): -- cgit v1.2.3 From c20d1d7d32ea2ab1d1c4dd9a34724a8732c23338 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 22:42:36 +1300 Subject: Extend unit tests for proxy.py to some tricky cases. --- libmproxy/proxy.py | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index a6a72d55..458ea2b5 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -326,11 +326,11 @@ class ProxyHandler(tcp.BaseHandler): if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) try: - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) @@ -356,31 +356,29 @@ class ProxyHandler(tcp.BaseHandler): line = self.get_line(self.rfile) if line == "": return None - if http.parse_init_connect(line): - r = http.parse_init_connect(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - host, port, httpversion = r - - headers = self.read_headers(authenticate=True) - self.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n'%self.server_version) + - '\r\n' - ) - self.wfile.flush() - dummycert = self.find_cert(client_conn, host, port, host) - try: + if not self.proxy_connect_state: + connparts = http.parse_init_connect(line) + if connparts: + host, port, httpversion = connparts + headers = self.read_headers(authenticate=True) + self.wfile.write( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n'%self.server_version) + + '\r\n' + ) + self.wfile.flush() + dummycert = self.find_cert(client_conn, host, port, host) sni = HandleSNI( self, client_conn, host, port, dummycert, self.config.certfile or self.config.cacert ) - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) - self.proxy_connect_state = (host, port, httpversion) - line = self.rfile.readline(line) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) + self.proxy_connect_state = (host, port, httpversion) + line = self.rfile.readline(line) if self.proxy_connect_state: r = http.parse_init_http(line) -- cgit v1.2.3 From 5c6587d4a80cc45b23154237ca94858da60c7da5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 10:37:06 +1300 Subject: Move HTTP auth module to netlib. --- libmproxy/proxy.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 458ea2b5..1bf57b8a 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -16,9 +16,8 @@ import sys, os, string, socket, time import shutil, tempfile, threading import SocketServer from OpenSSL import SSL -from netlib import odict, tcp, http, wsgi, certutils, http_status +from netlib import odict, tcp, http, wsgi, certutils, http_status, http_auth import utils, flow, version, platform, controller -import authentication KILL = 0 @@ -619,14 +618,14 @@ def process_proxy_options(parser, options): if len(options.auth_singleuser.split(':')) != 2: parser.error("Please specify user in the format username:password") username, password = options.auth_singleuser.split(':') - password_manager = authentication.SingleUserPasswordManager(username, password) + password_manager = http_auth.PassManSingleUser(username, password) elif options.auth_nonanonymous: - password_manager = authentication.PermissivePasswordManager() + password_manager = http_auth.PassManNonAnon() elif options.auth_htpasswd: - password_manager = authentication.HtpasswdPasswordManager(options.auth_htpasswd) - authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy") + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") else: - authenticator = authentication.NullProxyAuth(None) + authenticator = http_auth.NullProxyAuth(None) return ProxyConfig( certfile = options.cert, -- cgit v1.2.3 From d5876a12ed09940df77da823bf857437f7496524 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 11:58:57 +1300 Subject: Unit test proxy option parsing. --- libmproxy/proxy.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 1bf57b8a..9fe878a9 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -573,22 +573,19 @@ def process_proxy_options(parser, options): if options.cert: options.cert = os.path.expanduser(options.cert) if not os.path.exists(options.cert): - parser.error("Manually created certificate does not exist: %s"%options.cert) + return parser.error("Manually created certificate does not exist: %s"%options.cert) cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): certutils.dummy_ca(cacert) - if getattr(options, "cache", None) is not None: - options.cache = os.path.expanduser(options.cache) body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - parser.errror("Can't set both reverse proxy and transparent proxy.") + return parser.error("Can't set both reverse proxy and transparent proxy.") if options.transparent_proxy: if not platform.resolver: - parser.error("Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") trans = dict( resolver = platform.resolver(), sslports = TRANSPARENT_SSL_PORTS @@ -599,30 +596,33 @@ def process_proxy_options(parser, options): if options.reverse_proxy: rp = utils.parse_proxy_spec(options.reverse_proxy) if not rp: - parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) + return parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) else: rp = None if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): - parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) + return parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) if options.certdir: options.certdir = os.path.expanduser(options.certdir) if not os.path.exists(options.certdir) or not os.path.isdir(options.certdir): - parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) + return parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: - parser.error("Please specify user in the format username:password") + return parser.error("Invalid single-user specification. Please use the format username:password") username, password = options.auth_singleuser.split(':') password_manager = http_auth.PassManSingleUser(username, password) elif options.auth_nonanonymous: password_manager = http_auth.PassManNonAnon() elif options.auth_htpasswd: - password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + try: + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + except ValueError, v: + return parser.error(v.message) authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") else: authenticator = http_auth.NullProxyAuth(None) -- cgit v1.2.3 From 2465b8a376a7eb04eef6a1cce46dd11a8f1830d8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 12:13:33 +1300 Subject: 100% unit test coverage on proxy.py. Hallelujah! --- libmproxy/proxy.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 9fe878a9..75e195ea 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -504,10 +504,7 @@ class ProxyServer(tcp.TCPServer): def handle_connection(self, request, client_address): h = ProxyHandler(self.config, request, client_address, self, self.channel, self.server_version) h.handle() - try: - h.finish() - except tcp.NetLibDisconnect, e: - pass + h.finish() def handle_shutdown(self): self.config.certstore.cleanup() @@ -540,7 +537,7 @@ class DummyServer: def __init__(self, config): self.config = config - def start_slave(self, klass, channel): + def start_slave(self, *args): pass def shutdown(self): -- cgit v1.2.3 From cde66cd58470cd68a76a9d8b1022a45e99a5cd8d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 22:03:27 +1300 Subject: Fuzzing, and fixes for errors found with fuzzing. --- libmproxy/proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 75e195ea..7459fadf 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -237,6 +237,8 @@ class ProxyHandler(tcp.BaseHandler): continue else: raise + except http.HttpError, v: + raise ProxyError(502, "Invalid server response.") else: break @@ -278,7 +280,6 @@ class ProxyHandler(tcp.BaseHandler): ) else: self.log(cc, cc.error) - if isinstance(e, ProxyError): self.send_error(e.code, e.msg, e.headers) else: -- cgit v1.2.3 From 790ad468e4352419ef519401680f99ee3beb148d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Mar 2013 14:35:36 +1300 Subject: Fix bug that caused mis-identification of some HTTPS connections in transparent mode. --- libmproxy/proxy.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 7459fadf..3d55190d 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -323,17 +323,18 @@ class ProxyHandler(tcp.BaseHandler): if not orig: raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") host, port = orig - if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): + if port in self.config.transparent_proxy["sslports"]: scheme = "https" - dummycert = self.find_cert(client_conn, host, port, host) - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) - try: - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + if not self.ssl_established: + dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) else: scheme = "http" line = self.get_line(self.rfile) -- cgit v1.2.3