diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/dump.py | 6 | ||||
| -rw-r--r-- | libmproxy/filt.py | 2 | ||||
| -rw-r--r-- | libmproxy/flow.py | 121 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 8 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 159 | ||||
| -rw-r--r-- | libmproxy/proxy.py | 9 | ||||
| -rw-r--r-- | libmproxy/stateobject.py | 4 |
7 files changed, 165 insertions, 144 deletions
diff --git a/libmproxy/dump.py b/libmproxy/dump.py index e76ea1ce..7b54f7c1 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -42,14 +42,14 @@ class Options(object): def str_response(resp): r = "%s %s"%(resp.code, resp.msg) - if resp.is_replay(): + if resp.is_replay: r = "[replay] " + r return r def str_request(req, showhost): - if req.client_conn: - c = req.client_conn.address[0] + if req.flow.client_conn: + c = req.flow.client_conn.address.host else: c = "[replay]" r = "%s %s %s"%(c, req.method, req.get_url(showhost)) diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 6a0c3075..09be41b8 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -198,7 +198,7 @@ class FDomain(_Rex): code = "d" help = "Domain" def __call__(self, f): - return bool(re.search(self.expr, f.request.host, re.IGNORECASE)) + return bool(re.search(self.expr, f.request.host or f.server_conn.address.host, re.IGNORECASE)) class FUrl(_Rex): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 4032461d..b4b939c7 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,7 +8,8 @@ import types import tnetstring, filt, script, utils, encoding, proxy from email.utils import parsedate_tz, formatdate, mktime_tz from netlib import odict, http, certutils, wsgi -import controller, version, protocol +from .proxy import ClientConnection, ServerConnection +import controller, version, protocol, stateobject import app @@ -19,6 +20,123 @@ ODict = odict.ODict ODictCaseless = odict.ODictCaseless +class BackreferenceMixin(object): + """ + If an attribute from the _backrefattr tuple is set, + this mixin sets a reference back on the attribute object. + Example: + e = Error() + f = Flow() + f.error = e + assert f is e.flow + """ + _backrefattr = tuple() + + def __setattr__(self, key, value): + super(BackreferenceMixin, self).__setattr__(key, value) + if key in self._backrefattr and value is not None: + setattr(value, self._backrefname, self) + + +class Error(stateobject.SimpleStateObject): + """ + An Error. + + This is distinct from an HTTP error response (say, a code 500), which + is represented by a normal Response object. This class is responsible + for indicating errors that fall outside of normal HTTP communications, + like interrupted connections, timeouts, protocol errors. + + Exposes the following attributes: + + flow: Flow object + msg: Message describing the error + timestamp: Seconds since the epoch + """ + def __init__(self, msg, timestamp=None): + """ + @type msg: str + @type timestamp: float + """ + self.msg = msg + self.timestamp = timestamp or utils.timestamp() + + _stateobject_attributes = dict( + msg=str, + timestamp=float + ) + + @classmethod + def _from_state(cls, state): + f = cls(None) # the default implementation assumes an empty constructor. Override accordingly. + f._load_state(state) + return f + + def copy(self): + c = copy.copy(self) + return c + + +class Flow(stateobject.SimpleStateObject, BackreferenceMixin): + def __init__(self, conntype, client_conn, server_conn): + self.conntype = conntype + self.client_conn = client_conn + self.server_conn = server_conn + self.error = None + + _backrefattr = ("error",) + _backrefname = "flow" + + _stateobject_attributes = dict( + error=Error, + client_conn=ClientConnection, + server_conn=ServerConnection, + conntype=str + ) + + def _get_state(self): + d = super(Flow, self)._get_state() + d.update(version=version.IVERSION) + return d + + @classmethod + def _from_state(cls, state): + f = cls(None, None, None) + f._load_state(state) + return f + + def copy(self): + f = copy.copy(self) + if self.error: + f.error = self.error.copy() + return f + + def modified(self): + """ + Has this Flow been modified? + """ + if self._backup: + return self._backup != self._get_state() + else: + return False + + def backup(self, force=False): + """ + Save a backup of this Flow, which can be reverted to using a + call to .revert(). + """ + if not self._backup: + self._backup = self._get_state() + + def revert(self): + """ + Revert to the last backed up state. + """ + if self._backup: + self._load_state(self._backup) + self._backup = None + + class AppRegistry: def __init__(self): self.apps = {} @@ -542,6 +660,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None + # FIXME response = Response._from_state(flow.request, rflow.response._get_state()) response._set_replay() flow.response = response diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index ae0d99a6..da85500b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -30,10 +30,10 @@ class ProtocolHandler(object): from . import http, tcp -protocols = dict( - http = dict(handler=http.HTTPHandler, flow=http.HTTPFlow), - tcp = dict(handler=tcp.TCPHandler), -) +protocols = { + 'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow), + 'tcp': dict(handler=tcp.TCPHandler) +} # PyCharm type hinting behaves bad if this is a dict constructor... def _handler(conntype, connection_handler): diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 29cdf446..5faf78e0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -5,7 +5,9 @@ from netlib import http, tcp, http_status, odict from netlib.odict import ODict, ODictCaseless from . import ProtocolHandler, ConnectionTypeChange, KILL from .. import encoding, utils, version, filt, controller, stateobject -from ..proxy import ProxyError, ClientConnection, ServerConnection +from ..proxy import ProxyError +from ..flow import Flow, Error + HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" CONTENT_MISSING = 0 @@ -51,117 +53,11 @@ class decoded(object): if self.ce: self.o.encode(self.ce) -# FIXME: Move out of http -class BackreferenceMixin(object): - """ - If an attribute from the _backrefattr tuple is set, - this mixin sets a reference back on the attribute object. - Example: - e = Error() - f = Flow() - f.error = e - assert f is e.flow - """ - _backrefattr = tuple() - - def __setattr__(self, key, value): - super(BackreferenceMixin, self).__setattr__(key, value) - if key in self._backrefattr and value is not None: - setattr(value, self._backrefname, self) - -# FIXME: Move out of http -class Error(stateobject.SimpleStateObject): - """ - An Error. - - This is distinct from an HTTP error response (say, a code 500), which - is represented by a normal Response object. This class is responsible - for indicating errors that fall outside of normal HTTP communications, - like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - def __init__(self, msg, timestamp=None): - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def copy(self): - c = copy.copy(self) - return c - -# FIXME: Move out of http -class Flow(stateobject.SimpleStateObject, BackreferenceMixin): - def __init__(self, conntype, client_conn, server_conn, error): - self.conntype = conntype - self.client_conn = client_conn - self.server_conn = server_conn - self.error = error - - _backrefattr = ("error",) - _backrefname = "flow" - - _stateobject_attributes = dict( - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - conntype=str - ) - - def _get_state(self): - d = super(Flow, self)._get_state() - d.update(version=version.IVERSION) - return d - - @classmethod - def _from_state(cls, state): - f = cls(None, None, None, None) - f._load_state(state) - return f - - def copy(self): - f = copy.copy(self) - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self._get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self._get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self._load_state(self._backup) - self._backup = None - class HTTPMessage(stateobject.SimpleStateObject): def __init__(self): self.flow = None # Will usually set by backref mixin + """@type: HTTPFlow""" def get_decoded_content(self): """ @@ -397,7 +293,7 @@ class HTTPRequest(HTTPMessage): form = form or self.form_out if form == "asterisk" or \ - form == "origin": + form == "origin": request_line = '%s %s HTTP/%s.%s' % (self.method, self.path, self.httpversion[0], self.httpversion[1]) elif form == "authority": request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port, @@ -422,7 +318,9 @@ class HTTPRequest(HTTPMessage): ] ) if not 'host' in headers: - headers["Host"] = [utils.hostport(self.scheme, self.host, self.port)] + headers["Host"] = [utils.hostport(self.scheme, + self.host or self.flow.server_conn.address.host, + self.port or self.flow.server_conn.address.port)] if self.content: headers["Content-Length"] = [str(len(self.content))] @@ -442,7 +340,7 @@ class HTTPRequest(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise Exception("CONTENT_MISSING") # FIXME correct exception class + raise RuntimeError("Cannot assemble flow with CONTENT_MISSING") head = self._assemble_head(form) if self.content: return head + self.content @@ -481,7 +379,6 @@ class HTTPRequest(HTTPMessage): e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0] )] - def get_form_urlencoded(self): """ Retrieves the URL-encoded form data, returning an ODict object. @@ -542,15 +439,19 @@ class HTTPRequest(HTTPMessage): def get_url(self, hostheader=False): """ - Returns a URL string, constructed from the Request's URL compnents. + Returns a URL string, constructed from the Request's URL components. If hostheader is True, we use the value specified in the request Host header to construct the URL. """ + host = None if hostheader: - host = self.headers.get_first("host") or self.host - else: - host = self.host + host = self.headers.get_first("host") + if not host: + if self.host: + host = self.host + else: + host = self.flow.server_conn.address.host host = host.encode("idna") return utils.unparse_url(self.scheme, host, self.port, self.path).encode('ascii') @@ -678,7 +579,7 @@ class HTTPResponse(HTTPMessage): ) if self.content: headers["Content-Length"] = [str(len(self.content))] - elif 'Transfer-Encoding' in self.headers: # add content-length for chuncked transfer-encoding with no content + elif 'Transfer-Encoding' in self.headers: # add content-length for chuncked transfer-encoding with no content headers["Content-Length"] = ["0"] return str(headers) @@ -694,7 +595,7 @@ class HTTPResponse(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise Exception("CONTENT_MISSING") # FIXME correct exception class + raise RuntimeError("Cannot assemble flow with CONTENT_MISSING") head = self._assemble_head() if self.content: return head + self.content @@ -759,8 +660,8 @@ class HTTPResponse(HTTPMessage): cookies = [] for header in cookie_headers: pairs = [pair.partition("=") for pair in header.split(';')] - cookie_name = pairs[0][0] # the key of the first key/value pairs - cookie_value = pairs[0][2] # the value of the first key/value pairs + cookie_name = pairs[0][0] # the key of the first key/value pairs + cookie_value = pairs[0][2] # the value of the first key/value pairs cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in pairs[1:]} cookies.append((cookie_name, (cookie_value, cookie_parameters))) return dict(cookies) @@ -783,12 +684,12 @@ class HTTPFlow(Flow): intercepting: Is this flow currently being intercepted? """ - def __init__(self, client_conn, server_conn, error, request, response): - Flow.__init__(self, "http", client_conn, server_conn, error) - self.request = request - self.response = response + def __init__(self, client_conn, server_conn): + Flow.__init__(self, "http", client_conn, server_conn) + self.request = None + self.response = None - self.intercepting = False # FIXME: Should that rather be an attribute of Flow? + self.intercepting = False # FIXME: Should that rather be an attribute of Flow? self._backup = None _backrefattr = Flow._backrefattr + ("request", "response") @@ -801,7 +702,7 @@ class HTTPFlow(Flow): @classmethod def _from_state(cls, state): - f = cls(None, None, None, None, None) + f = cls(None, None) f._load_state(state) return f @@ -839,7 +740,7 @@ class HTTPFlow(Flow): self.request.reply(KILL) elif self.response and not self.response.reply.acked: self.response.reply(KILL) - master.handle_error(self) + master.handle_error(self.error) self.intercepting = False def intercept(self): @@ -932,7 +833,7 @@ class HTTPHandler(ProtocolHandler): self.process_request(flow.request) flow.response = self.get_response_from_server(flow.request) - self.c.log("response", [flow.response._assemble_response_line() if not LEGACY else flow.response._assemble().splitlines()[0]]) + self.c.log("response", [flow.response._assemble_response_line()]) response_reply = self.c.channel.ask("response" if LEGACY else "httpresponse", flow.response if LEGACY else flow) if response_reply is None or response_reply == KILL: @@ -982,7 +883,7 @@ class HTTPHandler(ProtocolHandler): else: err = message - self.c.log("error: %s" %err) + self.c.log("error: %s" % err) if flow: flow.error = Error(err) diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index e69bb6da..e3e40c7b 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -172,7 +172,7 @@ class RequestReplayThread(threading.Thread): ) self.channel.ask("response", response) except (ProxyError, http.HttpError, tcp.NetLibError), v: - err = flow.Error(self.flow.request, str(v)) + err = flow.Error(str(v)) self.channel.ask("error", err) """ @@ -291,9 +291,10 @@ class ConnectionHandler: A protocol handler must raise a ConnTypeChanged exception if it detects that this is happening """ # TODO: Implement SSL pass-through handling and change conntype - passthrough = ["echo.websocket.org", - "174.129.224.73" # echo.websocket.org, transparent mode - ] + passthrough = [ + "echo.websocket.org", + "174.129.224.73" # echo.websocket.org, transparent mode + ] if self.server_conn.address.host in passthrough or self.sni in passthrough: self.conntype = "tcp" return diff --git a/libmproxy/stateobject.py b/libmproxy/stateobject.py index ef8879b8..2cbec068 100644 --- a/libmproxy/stateobject.py +++ b/libmproxy/stateobject.py @@ -1,4 +1,4 @@ -class StateObject: +class StateObject(object): def _get_state(self): raise NotImplementedError @@ -56,7 +56,7 @@ class SimpleStateObject(StateObject): helper for _load_state. loads the given attribute from the state. """ - if state[attr] is None: + if state.get(attr, None) is None: setattr(self, attr, None) return |