diff options
| -rw-r--r-- | docs/Makefile | 2 | ||||
| -rw-r--r-- | docs/conf.py | 3 | ||||
| -rw-r--r-- | docs/dev/models.rst | 47 | ||||
| -rw-r--r-- | examples/dns_spoofing.py | 2 | ||||
| -rw-r--r-- | examples/redirect_requests.py | 9 | ||||
| -rw-r--r-- | libmproxy/console/common.py | 2 | ||||
| -rw-r--r-- | libmproxy/dump.py | 6 | ||||
| -rw-r--r-- | libmproxy/flow.py | 2 | ||||
| -rw-r--r-- | libmproxy/models/http.py | 65 | ||||
| -rw-r--r-- | test/test_dump.py | 2 | ||||
| -rw-r--r-- | test/test_flow.py | 10 | ||||
| -rw-r--r-- | test/test_server.py | 4 |
12 files changed, 81 insertions, 73 deletions
diff --git a/docs/Makefile b/docs/Makefile index a22bc8a2..99264d90 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -192,4 +192,4 @@ pseudoxml: @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." livehtml: - sphinx-autobuild -b html -z '../libmproxy' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
\ No newline at end of file + sphinx-autobuild -b html -z '../libmproxy' -z '../../netlib/netlib' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 1e686007..ff4a4a86 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,6 +40,9 @@ extensions = [ 'sphinxcontrib.documentedlist' ] +# https://github.com/sphinx-doc/sphinx/pull/2053 +napoleon_include_special_with_doc = False + autodoc_member_order = "bysource" # Add any paths that contain templates here, relative to this directory. diff --git a/docs/dev/models.rst b/docs/dev/models.rst index 9929e4b2..617bcb1f 100644 --- a/docs/dev/models.rst +++ b/docs/dev/models.rst @@ -8,18 +8,49 @@ Models are missing**. Please read the source code instead. -.. automodule:: libmproxy.models - :show-inheritance: - :members: - :undoc-members: +.. automodule:: netlib.http + + .. autoclass:: Request + + .. rubric:: Data + .. autoattribute:: first_line_format + .. autoattribute:: method + .. autoattribute:: scheme + .. autoattribute:: host + .. autoattribute:: port + .. autoattribute:: path + .. autoattribute:: http_version + .. autoattribute:: headers + .. autoattribute:: content + .. autoattribute:: timestamp_start + .. autoattribute:: timestamp_end + .. rubric:: Computed Properties and Convenience Methods + .. autoattribute:: text + .. autoattribute:: url + .. autoattribute:: pretty_host + .. autoattribute:: pretty_url + .. autoattribute:: query + .. autoattribute:: cookies + .. autoattribute:: path_components + .. automethod:: anticache + .. automethod:: anticomp + .. automethod:: constrain_encoding + .. autoattribute:: urlencoded_form + .. autoattribute:: multipart_form + + .. autoclass:: Response + .. warning:: Docs missing. -.. automodule:: netlib.http.semantics - :members: Request, Response - :undoc-members: .. autoclass:: Headers - :show-inheritance: :members: :special-members: :no-undoc-members: + + + .. autoclass:: decoded + +.. automodule:: libmproxy.models + :show-inheritance: + :members: HTTPFlow, Error, ClientConnection, ServerConnection
\ No newline at end of file diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py index 98495d45..7eb79695 100644 --- a/examples/dns_spoofing.py +++ b/examples/dns_spoofing.py @@ -39,7 +39,7 @@ def request(context, flow): sni = None port = 80 - host_header = flow.request.pretty_host(hostheader=True) + host_header = flow.request.pretty_host m = parse_host_header.match(host_header) if m: host_header = m.group("host").strip("[]") diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index ca24c42a..5cc81633 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -5,12 +5,12 @@ from libmproxy.models import HTTPResponse from netlib.http import Headers def request(context, flow): - # pretty_host(hostheader=True) takes the Host: header of the request into account, + # pretty_host takes the "Host" header of the request into account, # which is useful in transparent mode where we usually only have the IP # otherwise. # Method 1: Answer with a locally generated response - if flow.request.pretty_host(hostheader=True).endswith("example.com"): + if flow.request.pretty_host.endswith("example.com"): resp = HTTPResponse( [1, 1], 200, "OK", Headers(Content_Type="text/html"), @@ -18,6 +18,5 @@ def request(context, flow): flow.reply(resp) # Method 2: Redirect the request to a different server - if flow.request.pretty_host(hostheader=True).endswith("example.org"): - flow.request.host = "mitmproxy.org" - flow.request.update_host_header() + if flow.request.pretty_host.endswith("example.org"): + flow.request.host = "mitmproxy.org"
\ No newline at end of file diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 4a2310c9..48cb0f87 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -390,7 +390,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2, req_timestamp = f.request.timestamp_start, req_is_replay = f.request.is_replay, req_method = f.request.method, - req_url = f.request.pretty_url(hostheader=hostheader), + req_url = f.request.pretty_url if hostheader else f.request.url, err_msg = f.error.msg if f.error else None, resp_code = f.response.status_code if f.response else None, diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 3915d4c8..43af953c 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -241,7 +241,11 @@ class DumpMaster(flow.FlowMaster): DELETE="red" ).get(method.upper(), "magenta") method = click.style(method, fg=method_color, bold=True) - url = click.style(flow.request.pretty_url(self.showhost), bold=True) + if self.showhost: + url = flow.request.pretty_url + else: + url = flow.request.url + url = click.style(url, bold=True) line = "{stickycookie}{client} {method} {url}".format( stickycookie=stickycookie, diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 7d51b6d1..55a4dbcf 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -252,7 +252,7 @@ class ServerPlaybackState: ] if not self.ignore_content: - form_contents = r.get_form() + form_contents = r.urlencoded_form or r.multipart_form if self.ignore_payload_params and form_contents: key.extend( p for p in form_contents diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py index 5a83cb46..101df7e6 100644 --- a/libmproxy/models/http.py +++ b/libmproxy/models/http.py @@ -6,9 +6,8 @@ import time from libmproxy import utils from netlib import encoding -from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING +from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING, decoded from netlib.tcp import Address -from netlib.utils import native from .. import version, stateobject from .flow import Flow @@ -17,7 +16,6 @@ class MessageMixin(stateobject.StateObject): _stateobject_attributes = dict( http_version=bytes, headers=Headers, - body=bytes, timestamp_start=float, timestamp_end=float ) @@ -74,6 +72,9 @@ class MessageMixin(stateobject.StateObject): def copy(self): c = copy.copy(self) + if hasattr(self, "data"): # FIXME remove condition + c.data = copy.copy(self.data) + c.headers = self.headers.copy() return c @@ -147,7 +148,7 @@ class HTTPRequest(MessageMixin, Request): def __init__( self, - form_in, + first_line_format, method, scheme, host, @@ -155,14 +156,14 @@ class HTTPRequest(MessageMixin, Request): path, http_version, headers, - body, + content, timestamp_start=None, timestamp_end=None, form_out=None, ): Request.__init__( self, - form_in, + first_line_format, method, scheme, host, @@ -170,11 +171,11 @@ class HTTPRequest(MessageMixin, Request): path, http_version, headers, - body, + content, timestamp_start, timestamp_end, ) - self.form_out = form_out or form_in + self.form_out = form_out or first_line_format # FIXME remove # Have this request's cookies been modified by sticky cookies or auth? self.stickycookie = False @@ -185,7 +186,8 @@ class HTTPRequest(MessageMixin, Request): _stateobject_attributes = MessageMixin._stateobject_attributes.copy() _stateobject_attributes.update( - form_in=str, + content=bytes, + first_line_format=str, method=bytes, scheme=bytes, host=bytes, @@ -225,7 +227,7 @@ class HTTPRequest(MessageMixin, Request): @classmethod def wrap(self, request): req = HTTPRequest( - form_in=request.form_in, + first_line_format=request.form_in, method=request.method, scheme=request.scheme, host=request.host, @@ -233,7 +235,7 @@ class HTTPRequest(MessageMixin, Request): path=request.path, http_version=request.http_version, headers=request.headers, - body=request.body, + content=request.content, timestamp_start=request.timestamp_start, timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), @@ -311,6 +313,7 @@ class HTTPResponse(MessageMixin, Response): _stateobject_attributes = MessageMixin._stateobject_attributes.copy() _stateobject_attributes.update( + body=bytes, status_code=int, msg=bytes ) @@ -400,22 +403,20 @@ class HTTPResponse(MessageMixin, Response): class HTTPFlow(Flow): """ A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: + transaction. + Attributes: request: HTTPRequest object response: HTTPResponse object error: Error object server_conn: ServerConnection object client_conn: ClientConnection object + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? Note that it's possible for a Flow to have both a response and an error object. This might happen, for instance, when a response was received from the server, but there was an error sending it back to the client. - - The following additional attributes are exposed: - - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? """ def __init__(self, client_conn, server_conn, live=None): @@ -485,36 +486,6 @@ class HTTPFlow(Flow): return c -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get("content-encoding") - if ce: - ce = native(ce, "ascii", "ignore") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - if not self.o.decode(): - self.ce = None - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) - - def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ diff --git a/test/test_dump.py b/test/test_dump.py index 56d3edd2..9586a304 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -62,7 +62,7 @@ def test_contentview(get_content_view): class TestDumpMaster: def _cycle(self, m, content): - f = tutils.tflow(req=netlib.tutils.treq(body=content)) + f = tutils.tflow(req=netlib.tutils.treq(content=content)) l = Log("connect") l.reply = mock.MagicMock() m.handle_log(l) diff --git a/test/test_flow.py b/test/test_flow.py index b54b7508..0cd45576 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -338,7 +338,7 @@ class TestServerPlaybackState: assert s._hash(r) == s._hash(r2) -class TestFlow: +class TestFlow(object): def test_copy(self): f = tutils.tflow(resp=True) a0 = f.get_state() @@ -1017,10 +1017,10 @@ class TestRequest: r.port = 22 assert r.url == "https://address:22/path" - assert r.pretty_url(True) == "https://address:22/path" + assert r.pretty_url == "https://address:22/path" r.headers["Host"] = "foo.com" - assert r.pretty_url(False) == "https://address:22/path" - assert r.pretty_url(True) == "https://foo.com:22/path" + assert r.url == "https://address:22/path" + assert r.pretty_url == "https://foo.com:22/path" def test_path_components(self): r = HTTPRequest.wrap(netlib.tutils.treq()) @@ -1043,7 +1043,7 @@ class TestRequest: def test_getset_form_urlencoded(self): d = odict.ODict([("one", "two"), ("three", "four")]) - r = HTTPRequest.wrap(netlib.tutils.treq(body=netlib.utils.urlencode(d.lst))) + r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) r.headers["content-type"] = HDR_FORM_URLENCODED assert r.get_form_urlencoded() == d diff --git a/test/test_server.py b/test/test_server.py index 5f644c96..2a09d2e5 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -235,7 +235,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): # There's a race here, which means we can get any of a number of errors. # Rather than introduce yet another sleep into the test suite, we just # relax the Exception specification. - tutils.raises(Exception, p.request, "get:'%s'" % response) + with raises(Exception): + p.request("get:'%s'" % response) def test_reconnect(self): req = "get:'%s/p/200:b@1:da'" % self.server.urlbase @@ -573,7 +574,6 @@ class TestProxy(tservers.HTTPProxTest): recvd += len(connection.recv(5000)) connection.close() - print(self.master.state.view._list) first_flow = self.master.state.view[0] second_flow = self.master.state.view[1] assert first_flow.server_conn.timestamp_tcp_setup |