From f4b58ba495f5faefbfdd85ae15532cf36e2f74c9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Dec 2013 06:33:18 +0100 Subject: move CONTINUE checks into mitmproxy --- libmproxy/proxy.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 3098aff4..cb931d10 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -415,8 +415,9 @@ class ProxyHandler(tcp.BaseHandler): raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) method, scheme, host, port, path, httpversion = r headers = self.read_headers(authenticate=True) - content = http.read_http_body_request( - self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit + self.handle_expect_header(headers, httpversion) + content = http.read_http_body( + self.rfile, headers, self.config.body_size_limit, True ) return flow.Request( client_conn, httpversion, host, port, scheme, method, path, headers, content, @@ -446,14 +447,23 @@ class ProxyHandler(tcp.BaseHandler): raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) method, path, httpversion = r headers = self.read_headers(authenticate=False) - content = http.read_http_body_request( - self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit + self.handle_expect_header(headers, httpversion) + content = http.read_http_body( + self.rfile, headers, self.config.body_size_limit, True ) return flow.Request( client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp() ) + def handle_expect_header(self, headers, httpversion): + if "expect" in headers: + if "100-continue" in headers['expect'] and httpversion >= (1, 1): + #FIXME: Check if content-length is over limit + self.wfile.write('HTTP/1.1 100 Continue\r\n' + '\r\n') + del headers['expect'] + def read_headers(self, authenticate=False): headers = http.read_headers(self.rfile) if headers is None: -- cgit v1.2.3 From 1e07d9e6e7962922707fb0f384e30fd4d9461e2a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 4 Jan 2014 14:35:11 +1300 Subject: Move app mechanism to flow.py Disable apps while message passing is improved. --- libmproxy/proxy.py | 162 ++++++++++++++++++++++------------------------------- 1 file changed, 66 insertions(+), 96 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 609ffb62..9b300aa1 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -2,7 +2,7 @@ import sys, os, string, socket, time import shutil, tempfile, threading import SocketServer from OpenSSL import SSL -from netlib import odict, tcp, http, wsgi, certutils, http_status, http_auth +from netlib import odict, tcp, http, certutils, http_status, http_auth import utils, flow, version, platform, controller @@ -209,84 +209,77 @@ class ProxyHandler(tcp.BaseHandler): return cc.requestcount += 1 - app = self.server.apps.get(request) - if app: - err = app.serve(request, self.wfile) - if err: - self.log(cc, "Error in wsgi app.", err.split("\n")) - return + request_reply = self.channel.ask(request) + if request_reply is None or request_reply == KILL: + return + elif isinstance(request_reply, flow.Response): + request = False + response = request_reply + response_reply = self.channel.ask(response) else: - request_reply = self.channel.ask(request) - if request_reply is None or request_reply == KILL: - return - elif isinstance(request_reply, flow.Response): - request = False - response = request_reply - response_reply = self.channel.ask(response) + request = request_reply + if self.config.reverse_proxy: + scheme, host, port = self.config.reverse_proxy + elif self.config.forward_proxy: + scheme, host, port = self.config.forward_proxy else: - request = request_reply - if self.config.reverse_proxy: - scheme, host, port = self.config.reverse_proxy - elif self.config.forward_proxy: - scheme, host, port = self.config.forward_proxy - else: - scheme, host, port = request.scheme, request.host, request.port - - # If we've already pumped a request over this connection, - # it's possible that the server has timed out. If this is - # the case, we want to reconnect without sending an error - # to the client. - while 1: - sc = self.get_server_connection(cc, scheme, host, port, self.sni, request=request) - sc.send(request) - if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected) - request.tcp_setup_timestamp = sc.tcp_setup_timestamp - request.ssl_setup_timestamp = sc.ssl_setup_timestamp - sc.rfile.reset_timestamps() - try: - tsstart = utils.timestamp() - peername = sc.connection.getpeername() - if peername: - request.ip = peername[0] - httpversion, code, msg, headers, content = http.read_response( - sc.rfile, - request.method, - self.config.body_size_limit - ) - except http.HttpErrorConnClosed, v: - self.del_server_connection() - if sc.requestcount > 1: - continue - else: - raise - except http.HttpError, v: - raise ProxyError(502, "Invalid server response.") + scheme, host, port = request.scheme, request.host, request.port + + # If we've already pumped a request over this connection, + # it's possible that the server has timed out. If this is + # the case, we want to reconnect without sending an error + # to the client. + while 1: + sc = self.get_server_connection(cc, scheme, host, port, self.sni, request=request) + sc.send(request) + if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected) + request.tcp_setup_timestamp = sc.tcp_setup_timestamp + request.ssl_setup_timestamp = sc.ssl_setup_timestamp + sc.rfile.reset_timestamps() + try: + tsstart = utils.timestamp() + peername = sc.connection.getpeername() + if peername: + request.ip = peername[0] + httpversion, code, msg, headers, content = http.read_response( + sc.rfile, + request.method, + self.config.body_size_limit + ) + except http.HttpErrorConnClosed, v: + self.del_server_connection() + if sc.requestcount > 1: + continue else: - break - - response = flow.Response( - request, httpversion, code, msg, headers, content, sc.cert, - sc.rfile.first_byte_timestamp - ) - response_reply = self.channel.ask(response) - # Not replying to the server invalidates the server - # connection, so we terminate. - if response_reply == KILL: - sc.terminate() + raise + except http.HttpError, v: + raise ProxyError(502, "Invalid server response.") + else: + break + response = flow.Response( + request, httpversion, code, msg, headers, content, sc.cert, + sc.rfile.first_byte_timestamp + ) + response_reply = self.channel.ask(response) + # Not replying to the server invalidates the server + # connection, so we terminate. if response_reply == KILL: + sc.terminate() + + if response_reply == KILL: + return + else: + response = response_reply + self.send_response(response) + if request and http.connection_close(request.httpversion, request.headers): + return + # We could keep the client connection when the server + # connection needs to go away. However, we want to mimic + # behaviour as closely as possible to the client, so we + # disconnect. + if http.connection_close(response.httpversion, response.headers): return - else: - response = response_reply - self.send_response(response) - if request and http.connection_close(request.httpversion, request.headers): - return - # We could keep the client connection when the server - # connection needs to go away. However, we want to mimic - # behaviour as closely as possible to the client, so we - # disconnect. - if http.connection_close(response.httpversion, response.headers): - return except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e: if hasattr(e, "code"): cc.error = "%s: %s"%(e.code, e.msg) @@ -526,7 +519,6 @@ class ProxyServer(tcp.TCPServer): except socket.error, v: raise ProxyServerError('Error starting proxy server: ' + v.strerror) self.channel = None - self.apps = AppRegistry() def start_slave(self, klass, channel): slave = klass(channel, self) @@ -541,28 +533,6 @@ class ProxyServer(tcp.TCPServer): h.finish() -class AppRegistry: - def __init__(self): - self.apps = {} - - def add(self, app, domain, port): - """ - Add a WSGI app to the registry, to be served for requests to the - specified domain, on the specified port. - """ - self.apps[(domain, port)] = wsgi.WSGIAdaptor(app, domain, port, version.NAMEVERSION) - - def get(self, request): - """ - Returns an WSGIAdaptor instance if request matches an app, or None. - """ - if (request.host, request.port) in self.apps: - return self.apps[(request.host, request.port)] - if "host" in request.headers: - host = request.headers["host"][0] - return self.apps.get((host, request.port), None) - - class DummyServer: bound = False def __init__(self, config): -- cgit v1.2.3 From 45eab17e0c35b9527dd8f68364fa577c61f33551 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 4 Jan 2014 14:42:32 +1300 Subject: Decouple message type from message class name. --- libmproxy/proxy.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 9b300aa1..7f39a5c5 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -97,10 +97,10 @@ class RequestReplayThread(threading.Thread): self.flow.request, httpversion, code, msg, headers, content, server.cert, server.rfile.first_byte_timestamp ) - self.channel.ask(response) + self.channel.ask("response", response) except (ProxyError, http.HttpError, tcp.NetLibError), v: err = flow.Error(self.flow.request, str(v)) - self.channel.ask(err) + self.channel.ask("error", err) class HandleSNI: @@ -173,7 +173,7 @@ class ProxyHandler(tcp.BaseHandler): self.server_conn.require_request = False self.server_conn.conn_info = conn_info - self.channel.ask(self.server_conn) + self.channel.ask("serverconnect", self.server_conn) self.server_conn.connect() except tcp.NetLibError, v: raise ProxyError(502, v) @@ -187,7 +187,7 @@ class ProxyHandler(tcp.BaseHandler): def handle(self): cc = flow.ClientConnect(self.client_address) self.log(cc, "connect") - self.channel.ask(cc) + self.channel.ask("clientconnect", cc) while self.handle_request(cc) and not cc.close: pass cc.close = True @@ -199,7 +199,7 @@ class ProxyHandler(tcp.BaseHandler): [ "handled %s requests"%cc.requestcount] ) - self.channel.tell(cd) + self.channel.tell("clientdisconnect", cd) def handle_request(self, cc): try: @@ -209,13 +209,13 @@ class ProxyHandler(tcp.BaseHandler): return cc.requestcount += 1 - request_reply = self.channel.ask(request) + request_reply = self.channel.ask("request", request) if request_reply is None or request_reply == KILL: return elif isinstance(request_reply, flow.Response): request = False response = request_reply - response_reply = self.channel.ask(response) + response_reply = self.channel.ask("response", response) else: request = request_reply if self.config.reverse_proxy: @@ -261,7 +261,7 @@ class ProxyHandler(tcp.BaseHandler): request, httpversion, code, msg, headers, content, sc.cert, sc.rfile.first_byte_timestamp ) - response_reply = self.channel.ask(response) + response_reply = self.channel.ask("response", response) # Not replying to the server invalidates the server # connection, so we terminate. if response_reply == KILL: @@ -288,7 +288,7 @@ class ProxyHandler(tcp.BaseHandler): if request: err = flow.Error(request, cc.error) - self.channel.ask(err) + self.channel.ask("error", err) self.log( cc, cc.error, ["url: %s"%request.get_url()] @@ -308,7 +308,7 @@ class ProxyHandler(tcp.BaseHandler): msg.append(" -> "+i) msg = "\n".join(msg) l = Log(msg) - self.channel.tell(l) + self.channel.tell("log", l) def find_cert(self, cc, host, port, sni): if self.config.certfile: -- cgit v1.2.3 From a2261e3cf01121fb466bc91cd4117204187ba059 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 5 Jan 2014 10:58:53 +1300 Subject: Introduce file descriptor decorators for Request objects Which lets us enable the apps again, now running from flow.py --- libmproxy/proxy.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'libmproxy/proxy.py') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 7f39a5c5..69e5f411 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -6,6 +6,8 @@ from netlib import odict, tcp, http, certutils, http_status, http_auth import utils, flow, version, platform, controller +TRANSPARENT_SSL_PORTS = [443, 8443] + KILL = 0 @@ -425,10 +427,12 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request( + r = flow.Request( client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp() ) + r.set_live(self.rfile, self.wfile) + return r def _read_request_origin_form(self, client_conn, scheme, host, port): """ @@ -456,10 +460,12 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request( + r = flow.Request( client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp() ) + r.set_live(self.rfile, self.wfile) + return r def read_headers(self, authenticate=False): headers = http.read_headers(self.rfile) @@ -560,7 +566,6 @@ def certificate_option_group(parser): ) -TRANSPARENT_SSL_PORTS = [443, 8443] def process_proxy_options(parser, options): if options.cert: @@ -603,7 +608,9 @@ def process_proxy_options(parser, options): if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): - return parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) + return parser.error( + "Client certificate directory does not exist or is not a directory: %s"%options.clientcerts + ) if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): if options.auth_singleuser: -- cgit v1.2.3