From 393c1d74afaaed908ea842a7cf804e15a121e7fd Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 2 Jul 2016 18:56:22 +0530 Subject: fix lint issues --- mitmproxy/flow/master.py | 1 - test/mitmproxy/tservers.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index efb5d013..520f82e9 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -16,7 +16,6 @@ from mitmproxy.flow import modules from mitmproxy.onboarding import app from mitmproxy.protocol import http_replay from mitmproxy.proxy.config import HostMatcher -from netlib import strutils class FlowMaster(controller.Master): diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 51f4b4e2..0760cb53 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -11,8 +11,6 @@ import pathod.pathoc from mitmproxy import flow, controller from mitmproxy.cmdline import APP_HOST, APP_PORT -from netlib import strutils - testapp = flask.Flask(__name__) -- cgit v1.2.3 From 5b5a3ffa8e5650012fcf278146305aabb322b975 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 28 Jun 2016 17:35:56 +0530 Subject: netlib: condition on PY2 rather than on PY3 Let's just hope PY4 doesn't break this! --- netlib/strutils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/netlib/strutils.py b/netlib/strutils.py index 414b2e57..cfcdf485 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -19,12 +19,12 @@ def native(s, *encoding_opts): """ if not isinstance(s, (six.binary_type, six.text_type)): raise TypeError("%r is neither bytes nor unicode" % s) - if six.PY3: - if isinstance(s, six.binary_type): - return s.decode(*encoding_opts) - else: + if six.PY2: if isinstance(s, six.text_type): return s.encode(*encoding_opts) + else: + if isinstance(s, six.binary_type): + return s.decode(*encoding_opts) return s -- cgit v1.2.3 From 17b727321f659ce66d57269285c0a021db144e71 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 28 Jun 2016 17:40:13 +0530 Subject: Replace map + lambda with list comprehensions --- mitmproxy/flow/export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mitmproxy/flow/export.py b/mitmproxy/flow/export.py index f0ac02ab..bffe5291 100644 --- a/mitmproxy/flow/export.py +++ b/mitmproxy/flow/export.py @@ -4,7 +4,7 @@ import json import re from textwrap import dedent -from six.moves.urllib.parse import quote, quote_plus +from six.moves import urllib import netlib.http @@ -48,7 +48,7 @@ def python_code(flow): print(response.text) """).strip() - components = map(lambda x: quote(x, safe=""), flow.request.path_components) + components = [urllib.parse.quote(c, safe="") for c in flow.request.path_components] url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) args = "" @@ -126,7 +126,7 @@ def locust_code(flow): max_wait = 3000 """).strip() - components = map(lambda x: quote(x, safe=""), flow.request.path_components) + components = [urllib.parse.quote(c, safe="") for c in flow.request.path_components] file_name = "_".join(components) name = re.sub('\W|^(?=\d)', '_', file_name) url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) -- cgit v1.2.3 From f623b3d99b46f9cdeabdbea31614270cc1832f3b Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 28 Jun 2016 17:41:30 +0530 Subject: py3++: test_flow_export --- mitmproxy/flow/export.py | 41 +++++++++++++++------- .../data/test_flow_export/python_post_json.py | 4 +-- test/mitmproxy/test_flow_export.py | 6 ++-- tox.ini | 2 +- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/mitmproxy/flow/export.py b/mitmproxy/flow/export.py index bffe5291..2b0f8984 100644 --- a/mitmproxy/flow/export.py +++ b/mitmproxy/flow/export.py @@ -4,15 +4,26 @@ import json import re from textwrap import dedent +import six from six.moves import urllib import netlib.http +def _native(s): + if six.PY2: + if isinstance(s, six.text_type): + return s.encode() + else: + if isinstance(s, six.binary_type): + return s.decode() + return s + + def dictstr(items, indent): lines = [] for k, v in items: - lines.append(indent + "%s: %s,\n" % (repr(k), repr(v))) + lines.append(indent + "%s: %s,\n" % (repr(_native(k)), repr(_native(v)))) return "{\n%s}\n" % "".join(lines) @@ -20,7 +31,7 @@ def curl_command(flow): data = "curl " for k, v in flow.request.headers.fields: - data += "-H '%s:%s' " % (k, v) + data += "-H '%s:%s' " % (_native(k), _native(v)) if flow.request.method != "GET": data += "-X %s " % flow.request.method @@ -29,7 +40,7 @@ def curl_command(flow): data += "'%s'" % full_url if flow.request.content: - data += " --data-binary '%s'" % flow.request.content + data += " --data-binary '%s'" % _native(flow.request.content) return data @@ -69,7 +80,7 @@ def python_code(flow): data = "\njson = %s\n" % dictstr(sorted(json_obj.items()), " ") args += "\n json=json," else: - data = "\ndata = '''%s'''\n" % flow.request.body + data = "\ndata = '''%s'''\n" % _native(flow.request.content) args += "\n data=data," code = code.format( @@ -85,7 +96,7 @@ def python_code(flow): def raw_request(flow): data = netlib.http.http1.assemble_request(flow.request) - return data + return _native(data) def is_json(headers, content): @@ -127,16 +138,20 @@ def locust_code(flow): """).strip() components = [urllib.parse.quote(c, safe="") for c in flow.request.path_components] - file_name = "_".join(components) - name = re.sub('\W|^(?=\d)', '_', file_name) - url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) + name = re.sub('\W|^(?=\d)', '_', "_".join(components)) if name == "" or name is None: new_name = "_".join([str(flow.request.host), str(flow.request.timestamp_start)]) name = re.sub('\W|^(?=\d)', '_', new_name) + + url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) + args = "" headers = "" if flow.request.headers: - lines = [(k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]] + lines = [ + (_native(k), _native(v)) for k, v in flow.request.headers.fields + if _native(k).lower() not in ["host", "cookie"] + ] lines = [" '%s': '%s',\n" % (k, v) for k, v in lines] headers += "\n headers = {\n%s }\n" % "".join(lines) args += "\n headers=headers," @@ -148,8 +163,8 @@ def locust_code(flow): args += "\n params=params," data = "" - if flow.request.body: - data = "\n data = '''%s'''\n" % flow.request.body + if flow.request.content: + data = "\n data = '''%s'''\n" % _native(flow.request.content) args += "\n data=data," code = code.format( @@ -164,8 +179,8 @@ def locust_code(flow): host = flow.request.scheme + "://" + flow.request.host code = code.replace(host, "' + self.locust.host + '") - code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) + '") - code = code.replace(quote(host), "' + quote(self.locust.host) + '") + code = code.replace(urllib.parse.quote_plus(host), "' + quote_plus(self.locust.host) + '") + code = code.replace(urllib.parse.quote(host), "' + quote(self.locust.host) + '") code = code.replace("'' + ", "") return code diff --git a/test/mitmproxy/data/test_flow_export/python_post_json.py b/test/mitmproxy/data/test_flow_export/python_post_json.py index 6c1b9740..5ef110f3 100644 --- a/test/mitmproxy/data/test_flow_export/python_post_json.py +++ b/test/mitmproxy/data/test_flow_export/python_post_json.py @@ -8,8 +8,8 @@ headers = { json = { - u'email': u'example@example.com', - u'name': u'example', + 'email': 'example@example.com', + 'name': 'example', } diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 9a263b1b..33c5137a 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -21,15 +21,15 @@ def python_equals(testdata, text): def req_get(): - return netlib.tutils.treq(method='GET', content='', path=b"/path?a=foo&a=bar&b=baz") + return netlib.tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") def req_post(): - return netlib.tutils.treq(method='POST', headers=()) + return netlib.tutils.treq(method=b'POST', headers=()) def req_patch(): - return netlib.tutils.treq(method='PATCH', path=b"/path?query=param") + return netlib.tutils.treq(method=b'PATCH', path=b"/path?query=param") class TestExportCurlCommand(): diff --git a/tox.ini b/tox.ini index 899fffc6..508f0066 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From d3611777539471e53d4fdedf352ed755a4092415 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 3 Jul 2016 18:03:34 +0530 Subject: h2: move header parsing to netlib --- mitmproxy/protocol/http2.py | 30 ++-------------------------- netlib/http/http2/__init__.py | 2 ++ netlib/http/http2/utils.py | 37 ++++++++++++++++++++++++++++++++++ pathod/protocols/http2.py | 46 +++++++------------------------------------ 4 files changed, 48 insertions(+), 67 deletions(-) create mode 100644 netlib/http/http2/utils.py diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index b9a30c7e..f6261b6b 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -330,39 +330,13 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) if self.zombie: # pragma: no cover raise exceptions.Http2ProtocolException("Zombie Stream") - authority = self.request_headers.get(':authority', '') - method = self.request_headers.get(':method', 'GET') - scheme = self.request_headers.get(':scheme', 'https') - path = self.request_headers.get(':path', '/') - self.request_headers.clear(":method") - self.request_headers.clear(":scheme") - self.request_headers.clear(":path") - host = None - port = None - - if path == '*' or path.startswith("/"): - first_line_format = "relative" - elif method == 'CONNECT': # pragma: no cover - raise NotImplementedError("CONNECT over HTTP/2 is not implemented.") - else: # pragma: no cover - first_line_format = "absolute" - # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = netlib.http.url.parse(path) - - if authority: - host, _, port = authority.partition(':') - - if not host: - host = 'localhost' - if not port: - port = 443 if scheme == 'https' else 80 - port = int(port) - data = [] while self.request_data_queue.qsize() > 0: data.append(self.request_data_queue.get()) data = b"".join(data) + first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_headers) + return models.HTTPRequest( first_line_format, method, diff --git a/netlib/http/http2/__init__.py b/netlib/http/http2/__init__.py index 6a979a0d..60064190 100644 --- a/netlib/http/http2/__init__.py +++ b/netlib/http/http2/__init__.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division from netlib.http.http2 import framereader +from netlib.http.http2.utils import parse_headers __all__ = [ "framereader", + "parse_headers", ] diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py new file mode 100644 index 00000000..4d5f580c --- /dev/null +++ b/netlib/http/http2/utils.py @@ -0,0 +1,37 @@ +from netlib.http import url + + +def parse_headers(headers): + authority = headers.get(':authority', b'') + method = headers.get(':method', b'GET') + scheme = headers.get(':scheme', b'https') + path = headers.get(':path', b'/') + + headers.clear(":method") + headers.clear(":scheme") + headers.clear(":path") + + host = None + port = None + + if path == b'*' or path.startswith(b"/"): + first_line_format = "relative" + elif method == b'CONNECT': # pragma: no cover + raise NotImplementedError("CONNECT over HTTP/2 is not implemented.") + else: # pragma: no cover + first_line_format = "absolute" + # FIXME: verify if path or :host contains what we need + scheme, host, port, _ = url.parse(path) + + if authority: + host, _, port = authority.partition(b':') + + if not host: + host = b'localhost' + + if not port: + port = 443 if scheme == b'https' else 80 + + port = int(port) + + return first_line_format, method, scheme, host, port, path diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index c8728940..d94cd981 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -7,8 +7,7 @@ import hyperframe.frame from hpack.hpack import Encoder, Decoder from netlib import utils, strutils -from netlib.http import url -from netlib.http.http2 import framereader +from netlib.http import http2 import netlib.http.headers import netlib.http.response import netlib.http.request @@ -101,46 +100,15 @@ class HTTP2StateProtocol(object): timestamp_end = time.time() - authority = headers.get(':authority', b'') - method = headers.get(':method', 'GET') - scheme = headers.get(':scheme', 'https') - path = headers.get(':path', '/') - - headers.clear(":method") - headers.clear(":scheme") - headers.clear(":path") - - host = None - port = None - - if path == '*' or path.startswith("/"): - first_line_format = "relative" - elif method == 'CONNECT': - first_line_format = "authority" - if ":" in authority: - host, port = authority.split(":", 1) - else: - host = authority - else: - first_line_format = "absolute" - # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = url.parse(path) - scheme = scheme.decode('ascii') - host = host.decode('ascii') - - if host is None: - host = 'localhost' - if port is None: - port = 80 if scheme == 'http' else 443 - port = int(port) + first_line_format, method, scheme, host, port, path = http2.parse_headers(headers) request = netlib.http.request.Request( first_line_format, - method.encode('ascii'), - scheme.encode('ascii'), - host.encode('ascii'), + method, + scheme, + host, port, - path.encode('ascii'), + path, b"HTTP/2.0", headers, body, @@ -286,7 +254,7 @@ class HTTP2StateProtocol(object): def read_frame(self, hide=False): while True: - frm = framereader.http2_read_frame(self.tcp_handler.rfile) + frm = http2.framereader.http2_read_frame(self.tcp_handler.rfile) if not hide and self.dump_frames: # pragma no cover print(frm.human_readable("<<")) -- cgit v1.2.3 From 45aa2174e235f542d75d0305e7fb498b9b0585f9 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 3 Jul 2016 21:14:01 +0530 Subject: http2: Remove TestReadRequestConnect test --- test/pathod/test_protocols_http2.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/test/pathod/test_protocols_http2.py b/test/pathod/test_protocols_http2.py index e42c2858..8d7efc82 100644 --- a/test/pathod/test_protocols_http2.py +++ b/test/pathod/test_protocols_http2.py @@ -367,37 +367,6 @@ class TestReadRequestAbsolute(netlib_tservers.ServerTestBase): assert req.port == 22 -class TestReadRequestConnect(netlib_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - self.wfile.write( - codecs.decode('00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085', 'hex_codec')) - self.wfile.write( - codecs.decode('00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7', 'hex_codec')) - self.wfile.flush() - - ssl = True - - def test_connect(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_ssl() - protocol = HTTP2StateProtocol(c, is_server=True) - protocol.connection_preface_performed = True - - req = protocol.read_request(NotImplemented) - assert req.first_line_format == "authority" - assert req.method == "CONNECT" - assert req.host == "address" - assert req.port == 22 - - req = protocol.read_request(NotImplemented) - assert req.first_line_format == "authority" - assert req.method == "CONNECT" - assert req.host == "example.com" - assert req.port == 443 - - class TestReadResponse(netlib_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): -- cgit v1.2.3 From 23e295b37eb3aee9ebe7c88cc1ebb1dc2ffa9cf2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 3 Jul 2016 19:04:16 +0200 Subject: py3: fix bytes vs. str --- netlib/http/http2/utils.py | 8 ++++---- pathod/protocols/http2.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py index 4d5f580c..4c01952d 100644 --- a/netlib/http/http2/utils.py +++ b/netlib/http/http2/utils.py @@ -2,10 +2,10 @@ from netlib.http import url def parse_headers(headers): - authority = headers.get(':authority', b'') - method = headers.get(':method', b'GET') - scheme = headers.get(':scheme', b'https') - path = headers.get(':path', b'/') + authority = headers.get(':authority', '').encode() + method = headers.get(':method', 'GET').encode() + scheme = headers.get(':scheme', 'https').encode() + path = headers.get(':path', '/').encode() headers.clear(":method") headers.clear(":scheme") diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index d94cd981..5ad120de 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -181,10 +181,10 @@ class HTTP2StateProtocol(object): headers = request.headers.copy() if ':authority' not in headers: - headers.insert(0, b':authority', authority.encode('ascii')) - headers.insert(0, b':scheme', request.scheme.encode('ascii')) - headers.insert(0, b':path', request.path.encode('ascii')) - headers.insert(0, b':method', request.method.encode('ascii')) + headers.insert(0, ':authority', authority) + headers.insert(0, ':scheme', request.scheme) + headers.insert(0, ':path', request.path) + headers.insert(0, ':method', request.method) if hasattr(request, 'stream_id'): stream_id = request.stream_id @@ -397,7 +397,7 @@ class HTTP2StateProtocol(object): self._handle_unexpected_frame(frm) headers = netlib.http.headers.Headers( - (k.encode('ascii'), v.encode('ascii')) for k, v in self.decoder.decode(header_blocks) + [[k, v] for k, v in self.decoder.decode(header_blocks, raw=True)] ) return stream_id, headers, body -- cgit v1.2.3 From 69e20b34def39652a4eb60465092c27fd065a94e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 12 Jun 2016 12:21:19 +0200 Subject: bump h2 dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 564eb4d7..0de4ba32 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( "construct>=2.5.2, <2.6", "cryptography>=1.3, <1.5", "Flask>=0.10.1, <0.12", - "h2>=2.3.1, <3", + "h2>=2.4.0, <3", "html2text>=2016.1.8, <=2016.5.29", "hyperframe>=4.0.1, <5", "lxml>=3.5.0, <3.7", -- cgit v1.2.3 From aa1b20318216f066c0acb22b429b74b5ed1ce111 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 30 May 2016 14:56:51 +0200 Subject: http2: implement direct handling of priority --- mitmproxy/protocol/http2.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index f6261b6b..d848affa 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -5,7 +5,6 @@ import time import traceback import h2.exceptions -import hyperframe import six from h2 import connection from h2 import events @@ -55,12 +54,12 @@ class SafeH2Connection(connection.H2Connection): self.update_settings(new_settings) self.conn.send(self.data_to_send()) - def safe_send_headers(self, is_zombie, stream_id, headers): - # make sure to have a lock - if is_zombie(): # pragma: no cover - raise exceptions.Http2ProtocolException("Zombie Stream") - self.send_headers(stream_id, headers.fields) - self.conn.send(self.data_to_send()) + def safe_send_headers(self, is_zombie, stream_id, headers, **kwargs): + with self.lock: + if is_zombie(): # pragma: no cover + raise exceptions.Http2ProtocolException("Zombie Stream") + self.send_headers(stream_id, headers.fields, **kwargs) + self.conn.send(self.data_to_send()) def safe_send_body(self, is_zombie, stream_id, chunks): for chunk in chunks: @@ -141,6 +140,10 @@ class Http2Layer(base.Layer): headers = netlib.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) self.streams[eid].timestamp_start = time.time() + if event.priority_updated is not None: + self.streams[eid].priority_weight = event.priority_updated.weight + self.streams[eid].priority_depends_on = event.priority_updated.depends_on + self.streams[eid].priority_exclusive = event.priority_updated.exclusive self.streams[eid].start() elif isinstance(event, events.ResponseReceived): headers = netlib.http.Headers([[k, v] for k, v in event.headers]) @@ -184,7 +187,6 @@ class Http2Layer(base.Layer): self.client_conn.send(self.client_conn.h2.data_to_send()) self._kill_all_streams() return False - elif isinstance(event, events.PushedStreamReceived): # pushed stream ids should be unique and not dependent on race conditions # only the parent stream id must be looked up first @@ -210,9 +212,18 @@ class Http2Layer(base.Layer): if depends_on in self.streams.keys() and self.streams[depends_on].server_stream_id: depends_on = self.streams[depends_on].server_stream_id - # weight is between 1 and 256 (inclusive), but represented as uint8 (0 to 255) - frame = hyperframe.frame.PriorityFrame(stream_id, depends_on, event.weight - 1, event.exclusive) - self.server_conn.send(frame.serialize()) + self.streams[eid].priority_weight = event.weight + self.streams[eid].priority_depends_on = event.depends_on + self.streams[eid].priority_exclusive = event.exclusive + + with self.server_conn.h2.lock: + self.server_conn.h2.prioritize( + stream_id, + weight=event.weight, + depends_on=depends_on, + exclusive=event.exclusive + ) + self.server_conn.send(self.server_conn.h2.data_to_send()) elif isinstance(event, events.TrailersReceived): raise NotImplementedError() @@ -267,7 +278,7 @@ class Http2Layer(base.Layer): self._kill_all_streams() return - self._cleanup_streams() + self._cleanup_streams() except Exception as e: self.log(repr(e), "info") self.log(traceback.format_exc(), "debug") @@ -296,6 +307,10 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.response_queued_data_length = 0 self.response_data_finished = threading.Event() + self.priority_weight = None + self.priority_depends_on = None + self.priority_exclusive = None + @property def data_queue(self): if self.response_arrived.is_set(): @@ -394,6 +409,9 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.is_zombie, self.server_stream_id, headers, + priority_weight=self.priority_weight, + priority_depends_on=self.priority_depends_on, + priority_exclusive=self.priority_exclusive, ) except Exception as e: raise e -- cgit v1.2.3 From 47db3469590e7987898ea753215982bde09d4568 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 3 Jul 2016 13:25:29 +0200 Subject: improve end_stream handling To replicate requests as close as possible frame-by-frame. This fixes an issue with broken HTTP/2 implemenation by Akamai and Twitter, which raise an error if we send an empty DataFrame only to indicate END_STREAM. --- mitmproxy/protocol/http2.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index d848affa..f398630d 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -140,6 +140,7 @@ class Http2Layer(base.Layer): headers = netlib.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) self.streams[eid].timestamp_start = time.time() + self.streams[eid].no_body = (event.stream_ended is not None) if event.priority_updated is not None: self.streams[eid].priority_weight = event.priority_updated.weight self.streams[eid].priority_depends_on = event.priority_updated.depends_on @@ -307,6 +308,8 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.response_queued_data_length = 0 self.response_data_finished = threading.Event() + self.no_body = False + self.priority_weight = None self.priority_depends_on = None self.priority_exclusive = None @@ -409,6 +412,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.is_zombie, self.server_stream_id, headers, + end_stream=self.no_body, priority_weight=self.priority_weight, priority_depends_on=self.priority_depends_on, priority_exclusive=self.priority_exclusive, @@ -418,11 +422,13 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) finally: self.server_conn.h2.lock.release() - self.server_conn.h2.safe_send_body( - self.is_zombie, - self.server_stream_id, - message.body - ) + if not self.no_body: + self.server_conn.h2.safe_send_body( + self.is_zombie, + self.server_stream_id, + message.body + ) + if self.zombie: # pragma: no cover raise exceptions.Http2ProtocolException("Zombie Stream") -- cgit v1.2.3 From 64880e7ebd53ec4c079fdac6616d684c54e0fd79 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 5 Jul 2016 20:14:16 +0200 Subject: handle related events from h2 --- mitmproxy/protocol/http2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index f398630d..28d6a890 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -145,6 +145,7 @@ class Http2Layer(base.Layer): self.streams[eid].priority_weight = event.priority_updated.weight self.streams[eid].priority_depends_on = event.priority_updated.depends_on self.streams[eid].priority_exclusive = event.priority_updated.exclusive + self.streams[eid].handled_priority_event = event.priority_updated self.streams[eid].start() elif isinstance(event, events.ResponseReceived): headers = netlib.http.Headers([[k, v] for k, v in event.headers]) @@ -205,6 +206,11 @@ class Http2Layer(base.Layer): self.streams[event.pushed_stream_id].request_data_finished.set() self.streams[event.pushed_stream_id].start() elif isinstance(event, events.PriorityUpdated): + if self.streams[eid].handled_priority_event is event: + # This event was already handled during stream creation + # HeadersFrame + Priority information as RequestReceived + return True + stream_id = event.stream_id if stream_id in self.streams.keys() and self.streams[stream_id].server_stream_id: stream_id = self.streams[stream_id].server_stream_id @@ -313,6 +319,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.priority_weight = None self.priority_depends_on = None self.priority_exclusive = None + self.handled_priority_event = None @property def data_queue(self): -- cgit v1.2.3 From 50fdb015538a38544f22c5154dcb742eb942fa48 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 5 Jul 2016 20:59:02 +0200 Subject: prevent early priority changes --- mitmproxy/protocol/http2.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 28d6a890..19c7c604 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -206,10 +206,15 @@ class Http2Layer(base.Layer): self.streams[event.pushed_stream_id].request_data_finished.set() self.streams[event.pushed_stream_id].start() elif isinstance(event, events.PriorityUpdated): - if self.streams[eid].handled_priority_event is event: - # This event was already handled during stream creation - # HeadersFrame + Priority information as RequestReceived - return True + if eid in self.streams: + if self.streams[eid].handled_priority_event is event: + # This event was already handled during stream creation + # HeadersFrame + Priority information as RequestReceived + return True + if eid in self.streams: + self.streams[eid].priority_weight = event.weight + self.streams[eid].priority_depends_on = event.depends_on + self.streams[eid].priority_exclusive = event.exclusive stream_id = event.stream_id if stream_id in self.streams.keys() and self.streams[stream_id].server_stream_id: @@ -219,10 +224,6 @@ class Http2Layer(base.Layer): if depends_on in self.streams.keys() and self.streams[depends_on].server_stream_id: depends_on = self.streams[depends_on].server_stream_id - self.streams[eid].priority_weight = event.weight - self.streams[eid].priority_depends_on = event.depends_on - self.streams[eid].priority_exclusive = event.exclusive - with self.server_conn.h2.lock: self.server_conn.h2.prioritize( stream_id, -- cgit v1.2.3 From ba9851db027a5c1cc6060274fbaf8b76b57ecb46 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 5 Jul 2016 21:13:24 +0200 Subject: always recreate tox envs --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 339342ae..8a83b478 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -25,7 +25,7 @@ install: - "pip install -U tox" test_script: - - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb" + - ps: "tox --recreate -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb" deploy_script: ps: | -- cgit v1.2.3 From 05de3b759fe75e2f6839d8e855c304239f97eec1 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 5 Jun 2016 12:53:41 +0200 Subject: http2: remove forbidden headers E.g.: send_error_reponse sets a connection header --- mitmproxy/protocol/http2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 19c7c604..9515eef9 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -478,6 +478,9 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) def send_response_headers(self, response): headers = response.headers.copy() headers.insert(0, ":status", str(response.status_code)) + for forbidden_header in h2.utilities.CONNECTION_HEADERS: + if forbidden_header in headers: + del headers[forbidden_header] with self.client_conn.h2.lock: self.client_conn.h2.safe_send_headers( self.is_zombie, -- cgit v1.2.3 From 8d2042ea8ad7ce30ef8808c3c5e556adf4160cd3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 5 Jun 2016 13:14:23 +0200 Subject: http2: test throttling at MAX_CONCURRENT_STREAMS --- test/mitmproxy/test_protocol_http2.py | 74 ++++++++++++++++++++++++++++++++++- test/netlib/tservers.py | 12 +++--- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 932c8df2..89bb16c6 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -3,9 +3,10 @@ from __future__ import (absolute_import, print_function, division) import pytest -import traceback import os +import traceback import tempfile + import h2 from mitmproxy.proxy.config import ProxyConfig @@ -46,6 +47,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() + if 'h2_server_settings' in self.kwargs: + h2_conn.update_settings(self.kwargs['h2_server_settings']) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + done = False while not done: try: @@ -508,3 +514,69 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase): if len(self.master.state.flows) == 1: assert self.master.state.flows[0].response is None + + +@requires_alpn +class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): + + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class(h2_server_settings={h2.settings.MAX_CONCURRENT_STREAMS: 2}) + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('X-Stream-ID', str(event.stream_id)), + ]) + h2_conn.send_data(event.stream_id, b'Stream-ID {}'.format(event.stream_id)) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_max_concurrent_streams(self): + client, h2_conn = self._setup_connection() + new_streams = [1, 3, 5, 7, 9, 11] + for id in new_streams: + # this will exceed MAX_CONCURRENT_STREAMS on the server connection + # and cause mitmproxy to throttle stream creation to the server + self._send_request(client.wfile, h2_conn, stream_id=id, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('X-Stream-ID', str(id)), + ]) + + ended_streams = 0 + while ended_streams != len(new_streams): + try: + header, body = framereader.http2_read_raw_frame(client.rfile) + events = h2_conn.receive_data(b''.join([header, body])) + except: + break + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + ended_streams += 1 + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == len(new_streams) + for flow in self.master.state.flows: + assert flow.response.status_code == 200 + assert "Stream-ID" in flow.response.body diff --git a/test/netlib/tservers.py b/test/netlib/tservers.py index 803aaa72..666f97ac 100644 --- a/test/netlib/tservers.py +++ b/test/netlib/tservers.py @@ -24,7 +24,7 @@ class _ServerThread(threading.Thread): class _TServer(tcp.TCPServer): - def __init__(self, ssl, q, handler_klass, addr): + def __init__(self, ssl, q, handler_klass, addr, **kwargs): """ ssl: A dictionary of SSL parameters: @@ -42,6 +42,8 @@ class _TServer(tcp.TCPServer): self.q = q self.handler_klass = handler_klass + if self.handler_klass is not None: + self.handler_klass.kwargs = kwargs self.last_handler = None def handle_client_connection(self, request, client_address): @@ -89,16 +91,16 @@ class ServerTestBase(object): addr = ("localhost", 0) @classmethod - def setup_class(cls): + def setup_class(cls, **kwargs): cls.q = queue.Queue() - s = cls.makeserver() + s = cls.makeserver(**kwargs) cls.port = s.address.port cls.server = _ServerThread(s) cls.server.start() @classmethod - def makeserver(cls): - return _TServer(cls.ssl, cls.q, cls.handler, cls.addr) + def makeserver(cls, **kwargs): + return _TServer(cls.ssl, cls.q, cls.handler, cls.addr, **kwargs) @classmethod def teardown_class(cls): -- cgit v1.2.3 From 5fecb8c843a4d0874f96d19eb72ebdff66e2f19d Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 5 Jun 2016 15:30:58 +0200 Subject: http2: test fatal connection termination --- test/mitmproxy/test_protocol_http2.py | 53 ++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 89bb16c6..2eb0b120 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -4,8 +4,8 @@ from __future__ import (absolute_import, print_function, division) import pytest import os -import traceback import tempfile +import traceback import h2 @@ -580,3 +580,54 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): for flow in self.master.state.flows: assert flow.response.status_code == 200 assert "Stream-ID" in flow.response.body + + +@requires_alpn +class TestConnectionTerminated(_Http2TestBase, _Http2ServerBase): + + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class() + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.RequestReceived): + h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data='foobar') + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_connection_terminated(self): + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ]) + + done = False + connection_terminated_event = None + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + for event in events: + if isinstance(event, h2.events.ConnectionTerminated): + connection_terminated_event = event + done = True + except: + break + + assert len(self.master.state.flows) == 1 + assert connection_terminated_event is not None + assert connection_terminated_event.error_code == 5 + assert connection_terminated_event.last_stream_id == 42 + assert connection_terminated_event.additional_data == 'foobar' -- cgit v1.2.3 From 45a634783af41ffdc429445f1f712cebfb5f654c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 15:03:40 -0700 Subject: tox: dont make sdist when linting --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 899fffc6..9e690573 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] envlist = py27, py35, docs, lint +skipsdist = True [testenv] usedevelop = True @@ -26,6 +27,4 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] deps = flake8>=2.6.2, <3 usedevelop = False -skip_install = True -skipsdist = True commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test -- cgit v1.2.3 From d51cf543bb74755ed5dd17ed02859912ec557ef4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 15:11:32 -0700 Subject: remove clean_bin, clarify unicode handling --- examples/tcp_message.py | 14 ++++---- mitmproxy/contentviews.py | 9 +++-- netlib/strutils.py | 80 ++++++++++++++++++++++++++++---------------- netlib/websockets/frame.py | 2 +- pathod/log.py | 5 +-- test/netlib/test_strutils.py | 40 +++++++++++++--------- 6 files changed, 93 insertions(+), 57 deletions(-) diff --git a/examples/tcp_message.py b/examples/tcp_message.py index 78500c19..6eced0dc 100644 --- a/examples/tcp_message.py +++ b/examples/tcp_message.py @@ -17,9 +17,11 @@ def tcp_message(ctx, tcp_msg): is_modified = False if modified_msg == tcp_msg.message else True tcp_msg.message = modified_msg - print("[tcp_message{}] from {} {} to {} {}:\r\n{}".format( - " (modified)" if is_modified else "", - "client" if tcp_msg.sender == tcp_msg.client_conn else "server", - tcp_msg.sender.address, - "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", - tcp_msg.receiver.address, strutils.clean_bin(tcp_msg.message))) + print( + "[tcp_message{}] from {} {} to {} {}:\r\n{}".format( + " (modified)" if is_modified else "", + "client" if tcp_msg.sender == tcp_msg.client_conn else "server", + tcp_msg.sender.address, + "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", + tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message)) + ) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index de88c9ea..b7f4d3ec 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -160,7 +160,7 @@ class ViewRaw(View): content_types = [] def __call__(self, data, **metadata): - return "Raw", format_text(strutils.bytes_to_escaped_str(data)) + return "Raw", format_text(strutils.bytes_to_escaped_str(data, True)) class ViewHex(View): @@ -597,10 +597,9 @@ def safe_to_print(lines, encoding="utf8"): for line in lines: clean_line = [] for (style, text) in line: - try: - text = strutils.clean_bin(text.decode(encoding, "strict")) - except UnicodeDecodeError: - text = strutils.clean_bin(text) + if isinstance(text, bytes): + text = text.decode(encoding, "replace") + text = strutils.escape_control_characters(text) clean_line.append((style, text)) yield clean_line diff --git a/netlib/strutils.py b/netlib/strutils.py index 414b2e57..4beb6ffd 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -1,4 +1,4 @@ -import unicodedata +import re import codecs import six @@ -28,51 +28,71 @@ def native(s, *encoding_opts): return s -def clean_bin(s, keep_spacing=True): - # type: (Union[bytes, six.text_type], bool) -> six.text_type +# Translate control characters to "safe" characters. This implementation initially +# replaced them with the matching control pictures (http://unicode.org/charts/PDF/U2400.pdf), +# but that turned out to render badly with monospace fonts. We are back to "." therefore. +_control_char_trans = { + x: ord(".") # x + 0x2400 for unicode control group pictures + for x in range(32) +} +_control_char_trans[127] = ord(".") # 0x2421 +_control_char_trans_newline = _control_char_trans.copy() +for x in ("\r", "\n", "\t"): + del _control_char_trans_newline[ord(x)] + + +if six.PY2: + pass +else: + _control_char_trans = str.maketrans(_control_char_trans) + _control_char_trans_newline = str.maketrans(_control_char_trans_newline) + + +def escape_control_characters(text, keep_spacing=True): """ - Cleans binary data to make it safe to display. + Replace all unicode C1 control characters from the given text with their respective control pictures. + For example, a null byte is replaced with the unicode character "\u2400". - Args: - keep_spacing: If False, tabs and newlines will also be replaced. + Args: + keep_spacing: If True, tabs and newlines will not be replaced. """ - if isinstance(s, six.text_type): - if keep_spacing: - keep = u" \n\r\t" - else: - keep = u" " + # type: (six.text_type) -> six.text_type + if not isinstance(text, six.text_type): + raise ValueError("text type must be unicode but is {}".format(type(text).__name__)) + + trans = _control_char_trans_newline if keep_spacing else _control_char_trans + if six.PY2: return u"".join( - ch if (unicodedata.category(ch)[0] not in "CZ" or ch in keep) else u"." - for ch in s - ) - else: - if keep_spacing: - keep = (9, 10, 13) # \t, \n, \r, - else: - keep = () - return "".join( - chr(ch) if (31 < ch < 127 or ch in keep) else "." - for ch in six.iterbytes(s) + six.unichr(trans.get(ord(ch), ord(ch))) + for ch in text ) + return text.translate(trans) -def bytes_to_escaped_str(data): +def bytes_to_escaped_str(data, keep_spacing=False): """ Take bytes and return a safe string that can be displayed to the user. Single quotes are always escaped, double quotes are never escaped: "'" + bytes_to_escaped_str(...) + "'" gives a valid Python string. + + Args: + keep_spacing: If True, tabs and newlines will not be escaped. """ - # TODO: We may want to support multi-byte characters without escaping them. - # One way to do would be calling .decode("utf8", "backslashreplace") first - # and then escaping UTF8 control chars (see clean_bin). if not isinstance(data, bytes): raise ValueError("data must be bytes, but is {}".format(data.__class__.__name__)) # We always insert a double-quote here so that we get a single-quoted string back # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their - return repr(b'"' + data).lstrip("b")[2:-1] + ret = repr(b'"' + data).lstrip("b")[2:-1] + if keep_spacing: + ret = re.sub( + r"(??@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) + assert ( + strutils.escape_control_characters(bytes(bytearray(range(128))).decode(), False) == + u'................................ !"#$%&\'()*+,-./0123456789:;<' + u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) def test_bytes_to_escaped_str(): @@ -37,6 +40,14 @@ def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"'") == r"\'" assert strutils.bytes_to_escaped_str(b'"') == r'"' + assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" + assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" + + assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" + assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace(" ", "") + with tutils.raises(ValueError): strutils.bytes_to_escaped_str(u"such unicode") @@ -45,10 +56,9 @@ def test_escaped_str_to_bytes(): assert strutils.escaped_str_to_bytes("foo") == b"foo" assert strutils.escaped_str_to_bytes("\x08") == b"\b" assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes("ü") == b'\xc3\xbc' assert strutils.escaped_str_to_bytes(u"\\x08") == b"\b" assert strutils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes(u"ü") == b'\xc3\xbc' + assert strutils.escaped_str_to_bytes(u"\u00fc") == b'\xc3\xbc' if six.PY2: with tutils.raises(ValueError): -- cgit v1.2.3 From 882b8b5d80e308599bdb602a390edd8d2be065e4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 15:58:52 -0700 Subject: don't cache tox on CI --- .appveyor.yml | 3 +-- .travis.yml | 6 +----- tox.ini | 2 -- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8a83b478..61817a92 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -25,7 +25,7 @@ install: - "pip install -U tox" test_script: - - ps: "tox --recreate -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb" + - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb" deploy_script: ps: | @@ -41,7 +41,6 @@ deploy_script: cache: - C:\Users\appveyor\AppData\Local\pip\cache - - C:\projects\mitmproxy\.tox notifications: - provider: Slack diff --git a/.travis.yml b/.travis.yml index 29d0897c..345d6a58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,8 +71,4 @@ notifications: on_failure: change on_start: never -cache: - directories: - - $HOME/build/mitmproxy/mitmproxy/.tox - - $HOME/.cache/pip - - $HOME/.pyenv +cache: pip diff --git a/tox.ini b/tox.ini index cab23b66..bbfa438f 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,6 @@ envlist = py27, py35, docs, lint skipsdist = True [testenv] -usedevelop = True deps = {env:CI_DEPS:} -rrequirements.txt @@ -26,5 +25,4 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] deps = flake8>=2.6.2, <3 -usedevelop = False commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test -- cgit v1.2.3 From 72f3b2bb17763890be2d014dce1c1e17b614ed92 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 19:32:25 -0700 Subject: minor code improvement --- mitmproxy/stateobject.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 8db6cda3..4074fbbf 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -52,19 +52,20 @@ class StateObject(netlib.basetypes.Serializable): """ state = state.copy() for attr, cls in six.iteritems(self._stateobject_attributes): + val = state.pop(attr) if state.get(attr) is None: - setattr(self, attr, state.pop(attr)) + setattr(self, attr, val) else: curr = getattr(self, attr) if hasattr(curr, "set_state"): - curr.set_state(state.pop(attr)) + curr.set_state(val) elif hasattr(cls, "from_state"): - obj = cls.from_state(state.pop(attr)) + obj = cls.from_state(val) setattr(self, attr, obj) elif _is_list(cls): cls = cls.__parameters__[0] if cls.__parameters__ else cls.__args__[0] - setattr(self, attr, [cls.from_state(x) for x in state.pop(attr)]) + setattr(self, attr, [cls.from_state(x) for x in val]) else: # primitive types such as int, str, ... - setattr(self, attr, cls(state.pop(attr))) + setattr(self, attr, cls(val)) if state: raise RuntimeWarning("Unexpected State in __setstate__: {}".format(state)) -- cgit v1.2.3 From 38fd1d3ad77a08b7299b3c0319c522bc6b78f09e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 12:14:48 -0700 Subject: fix issue introduced in 72f3b2b --- mitmproxy/stateobject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 4074fbbf..5e4ae6e3 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -53,7 +53,7 @@ class StateObject(netlib.basetypes.Serializable): state = state.copy() for attr, cls in six.iteritems(self._stateobject_attributes): val = state.pop(attr) - if state.get(attr) is None: + if val is None: setattr(self, attr, val) else: curr = getattr(self, attr) -- cgit v1.2.3 From 811b72cd304a8c75efaf706fd57cfbe9494cd3d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 13:01:24 -0700 Subject: blacklist pysftp 0.2.9 https://bitbucket.org/dundeemt/pysftp/issues/97/pysftp-029-breaks-builds --- release/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/setup.py b/release/setup.py index 9876af0a..78155140 100644 --- a/release/setup.py +++ b/release/setup.py @@ -10,7 +10,7 @@ setup( "virtualenv>=14.0.5, <14.1", "wheel>=0.29.0, <0.30", "six>=1.10.0, <1.11", - "pysftp>=0.2.8, <0.3", + "pysftp>=0.2.8, !=0.2.9, <0.3", ], entry_points={ "console_scripts": [ -- cgit v1.2.3 From 8b564bc934b8b51734c41192ecb0135c264087cc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 16:47:32 -0700 Subject: fix pathod log encoding --- netlib/strutils.py | 4 ++-- pathod/log.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/netlib/strutils.py b/netlib/strutils.py index 809f5e17..59816dba 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -156,8 +156,8 @@ def hexdump(s): part = s[i:i + 16] x = " ".join("{:0=2x}".format(i) for i in six.iterbytes(part)) x = x.ljust(47) # 16*2 + 15 - part_repr = escape_control_characters( + part_repr = native(escape_control_characters( part.decode("ascii", "replace").replace(u"\ufffd", u"."), False - ) + )) yield (offset, x, part_repr) diff --git a/pathod/log.py b/pathod/log.py index d39496e0..47837101 100644 --- a/pathod/log.py +++ b/pathod/log.py @@ -62,9 +62,15 @@ class LogCtx(object): for line in strutils.hexdump(data): self("\t%s %s %s" % line) else: - data = data.decode("ascii", "replace").replace(u"\ufffd", u".") - for i in strutils.escape_control_characters(data).split(u"\n"): - self(u"\t%s" % i) + data = strutils.native( + strutils.escape_control_characters( + data + .decode("ascii", "replace") + .replace(u"\ufffd", u".") + ) + ) + for i in data.split("\n"): + self("\t%s" % i) def __call__(self, line): self.lines.append(line) -- cgit v1.2.3 From 444f0a4c397e3d664ce80f65d176d871b8e1194e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 17:31:08 -0700 Subject: py3++ --- examples/custom_contentviews.py | 4 ++-- examples/redirect_requests.py | 5 +++-- mitmproxy/contentviews.py | 6 +++--- netlib/strutils.py | 25 +++++++++---------------- test/mitmproxy/test_examples.py | 10 +++++----- test/netlib/test_strutils.py | 15 ++++----------- 6 files changed, 26 insertions(+), 39 deletions(-) diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py index 05ebeb69..92fb6a58 100644 --- a/examples/custom_contentviews.py +++ b/examples/custom_contentviews.py @@ -11,7 +11,7 @@ class ViewPigLatin(contentviews.View): content_types = ["text/html"] def __call__(self, data, **metadata): - if strutils.isXML(data): + if strutils.is_xml(data): parser = lxml.etree.HTMLParser( strip_cdata=True, remove_blank_text=True @@ -20,7 +20,7 @@ class ViewPigLatin(contentviews.View): docinfo = d.getroottree().docinfo def piglify(src): - words = string.split(src) + words = src.split() ret = '' for word in words: idx = -1 diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index d7db3f1c..af2aa907 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -13,9 +13,10 @@ def request(context, flow): # Method 1: Answer with a locally generated response if flow.request.pretty_host.endswith("example.com"): resp = HTTPResponse( - "HTTP/1.1", 200, "OK", + b"HTTP/1.1", 200, b"OK", Headers(Content_Type="text/html"), - "helloworld") + b"helloworld" + ) flow.reply.send(resp) # Method 2: Redirect the request to a different server diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index b7f4d3ec..6f64e360 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -143,11 +143,11 @@ class ViewAuto(View): ct = "%s/%s" % (ct[0], ct[1]) if ct in content_types_map: return content_types_map[ct][0](data, **metadata) - elif strutils.isXML(data.decode()): + elif strutils.is_xml(data): return get("XML")(data, **metadata) if metadata.get("query"): return get("Query")(data, **metadata) - if data and strutils.isMostlyBin(data.decode()): + if data and strutils.is_mostly_bin(data): return get("Hex")(data) if not data: return "No content", [] @@ -240,7 +240,7 @@ class ViewHTML(View): content_types = ["text/html"] def __call__(self, data, **metadata): - if strutils.isXML(data.decode()): + if strutils.is_xml(data): parser = lxml.etree.HTMLParser( strip_cdata=True, remove_blank_text=True diff --git a/netlib/strutils.py b/netlib/strutils.py index 809f5e17..7ad15c96 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -114,24 +114,17 @@ def escaped_str_to_bytes(data): return codecs.escape_decode(data)[0] -def isBin(s): - """ - Does this string have any non-ASCII characters? - """ - for i in s: - i = ord(i) - if i < 9 or 13 < i < 32 or 126 < i: - return True - return False - - -def isMostlyBin(s): - s = s[:100] - return sum(isBin(ch) for ch in s) / len(s) > 0.3 +def is_mostly_bin(s): + # type: (bytes) -> bool + return sum( + i < 9 or 13 < i < 32 or 126 < i + for i in six.iterbytes(s[:100]) + ) > 30 -def isXML(s): - return s.strip().startswith("<") +def is_xml(s): + # type: (bytes) -> bool + return s.strip().startswith(b"<") def clean_hanging_newline(t): diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 607d6faf..22d3c425 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -73,9 +73,9 @@ def test_add_header(): def test_custom_contentviews(): with example("custom_contentviews.py") as ex: pig = ex.ctx.contentview - _, fmt = pig("test!") - assert any('esttay!' in val[0][1] for val in fmt) - assert not pig("gobbledygook") + _, fmt = pig(b"test!") + assert any(b'esttay!' in val[0][1] for val in fmt) + assert not pig(b"gobbledygook") def test_iframe_injector(): @@ -103,7 +103,7 @@ def test_modify_form(): def test_modify_querystring(): - flow = tutils.tflow(req=netutils.treq(path="/search?q=term")) + flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term")) with example("modify_querystring.py") as ex: ex.run("request", flow) assert flow.request.query["mitmproxy"] == "rocks" @@ -126,7 +126,7 @@ def test_modify_response_body(): def test_redirect_requests(): - flow = tutils.tflow(req=netutils.treq(host="example.org")) + flow = tutils.tflow(req=netutils.treq(host=b"example.org")) with example("redirect_requests.py") as ex: ex.run("request", flow) assert flow.request.host == "mitmproxy.org" diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 33bce714..f88e33ed 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -68,17 +68,10 @@ def test_escaped_str_to_bytes(): strutils.escaped_str_to_bytes(b"very byte") -def test_isBin(): - assert not strutils.isBin("testing\n\r") - assert strutils.isBin("testing\x01") - assert strutils.isBin("testing\x0e") - assert strutils.isBin("testing\x7f") - - -def test_isXml(): - assert not strutils.isXML("foo") - assert strutils.isXML(" Date: Wed, 6 Jul 2016 19:50:06 -0700 Subject: py3++, multidict fixes This commit improves Python 3 compatibility and fixes two multidict issues: 1. Headers.items(multi=True) now decodes fields 2. MultiDict.clear(item) has been removed, as Python's MutableMapping already defines .clear() with different semantics. This is confusing for everyone who expects a dict-like object. `.pop("attr", None)` is not fantastic, but it's the Python way to do it. --- mitmproxy/protocol/http2.py | 2 +- mitmproxy/web/app.py | 65 ++++++++++++++++++++++++++------------- mitmproxy/web/master.py | 4 +-- netlib/http/headers.py | 9 ++++++ netlib/http/http2/utils.py | 6 ++-- netlib/multidict.py | 24 +++++---------- test/mitmproxy/test_web_master.py | 2 +- tox.ini | 2 +- 8 files changed, 68 insertions(+), 46 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 9515eef9..b6623aa3 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -448,7 +448,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) status_code = int(self.response_headers.get(':status', 502)) headers = self.response_headers.copy() - headers.clear(":status") + headers.pop(":status", None) return models.HTTPResponse( http_version=b"HTTP/2.0", diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index a2798472..f9d0dca6 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -12,34 +12,57 @@ from io import BytesIO from mitmproxy.flow import FlowWriter, FlowReader from mitmproxy import filt +from mitmproxy import models from netlib import version -def _strip_content(flow_state): +def convert_flow_to_json_dict(flow): + # type: (models.Flow) -> dict """ Remove flow message content and cert to save transmission space. Args: - flow_state: The original flow state. Will be left unmodified + flow: The original flow. """ - for attr in ("request", "response"): - if attr in flow_state: - message = flow_state[attr] - if message is None: - continue - if message["content"]: - message["contentLength"] = len(message["content"]) - else: - message["contentLength"] = None - del message["content"] - - if "backup" in flow_state: - del flow_state["backup"] - flow_state["modified"] = True - - flow_state.get("server_conn", {}).pop("cert", None) - - return flow_state + f = { + "id": flow.id, + "intercepted": flow.intercepted, + "client_conn": flow.client_conn.get_state(), + "server_conn": flow.server_conn.get_state(), + "type": flow.type + } + if flow.error: + f["error"] = flow.error.get_state() + + if isinstance(flow, models.HTTPFlow): + if flow.request: + f["request"] = { + "method": flow.request.method, + "scheme": flow.request.scheme, + "host": flow.request.host, + "port": flow.request.port, + "path": flow.request.path, + "http_version": flow.request.http_version, + "headers": tuple(flow.request.headers.items(True)), + "contentLength": len(flow.request.content) if flow.request.content is not None else None, + "timestamp_start": flow.request.timestamp_start, + "timestamp_end": flow.request.timestamp_end, + "is_replay": flow.request.is_replay, + } + if flow.response: + f["response"] = { + "http_version": flow.response.http_version, + "status_code": flow.response.status_code, + "reason": flow.response.reason, + "headers": tuple(flow.response.headers.items(True)), + "contentLength": len(flow.response.content) if flow.response.content is not None else None, + "timestamp_start": flow.response.timestamp_start, + "timestamp_end": flow.response.timestamp_end, + "is_replay": flow.response.is_replay, + } + f.get("server_conn", {}).pop("cert", None) + + return f class APIError(tornado.web.HTTPError): @@ -158,7 +181,7 @@ class Flows(RequestHandler): def get(self): self.write(dict( - data=[_strip_content(f.get_state()) for f in self.state.flows] + data=[convert_flow_to_json_dict(f) for f in self.state.flows] )) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index d034a24b..737bb95f 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -27,7 +27,7 @@ class WebFlowView(flow.FlowView): app.ClientConnection.broadcast( type="UPDATE_FLOWS", cmd="add", - data=app._strip_content(f.get_state()) + data=app.convert_flow_to_json_dict(f) ) def _update(self, f): @@ -35,7 +35,7 @@ class WebFlowView(flow.FlowView): app.ClientConnection.broadcast( type="UPDATE_FLOWS", cmd="update", - data=app._strip_content(f.get_state()) + data=app.convert_flow_to_json_dict(f) ) def _remove(self, f): diff --git a/netlib/http/headers.py b/netlib/http/headers.py index f052a53b..413add87 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -148,6 +148,15 @@ class Headers(multidict.MultiDict): value = _always_bytes(value) super(Headers, self).insert(index, key, value) + def items(self, multi=False): + if multi: + return ( + (_native(k), _native(v)) + for k, v in self.fields + ) + else: + return super(Headers, self).items() + def replace(self, pattern, repl, flags=0): """ Replaces a regular expression pattern with repl in each "name: value" diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py index 4c01952d..164bacc8 100644 --- a/netlib/http/http2/utils.py +++ b/netlib/http/http2/utils.py @@ -7,9 +7,9 @@ def parse_headers(headers): scheme = headers.get(':scheme', 'https').encode() path = headers.get(':path', '/').encode() - headers.clear(":method") - headers.clear(":scheme") - headers.clear(":path") + headers.pop(":method", None) + headers.pop(":scheme", None) + headers.pop(":path", None) host = None port = None diff --git a/netlib/multidict.py b/netlib/multidict.py index 50c879d9..51053ff6 100644 --- a/netlib/multidict.py +++ b/netlib/multidict.py @@ -170,18 +170,10 @@ class _MultiDict(MutableMapping, basetypes.Serializable): else: return super(_MultiDict, self).items() - def clear(self, key): - """ - Removes all items with the specified key, and does not raise an - exception if the key does not exist. - """ - if key in self: - del self[key] - def collect(self): """ Returns a list of (key, value) tuples, where values are either - singular if threre is only one matching item for a key, or a list + singular if there is only one matching item for a key, or a list if there are more than one. The order of the keys matches the order in the underlying fields list. """ @@ -204,18 +196,16 @@ class _MultiDict(MutableMapping, basetypes.Serializable): .. code-block:: python # Simple dict with duplicate values. - >>> d - MultiDictView[("name", "value"), ("a", "false"), ("a", "42")] + >>> d = MultiDict([("name", "value"), ("a", False), ("a", 42)]) >>> d.to_dict() { "name": "value", - "a": ["false", "42"] + "a": [False, 42] } """ - d = {} - for k, v in self.collect(): - d[k] = v - return d + return { + k: v for k, v in self.collect() + } def get_state(self): return self.fields @@ -307,4 +297,4 @@ class MultiDictView(_MultiDict): @fields.setter def fields(self, value): - return self._setter(value) + self._setter(value) diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 98f53c93..f0fafe24 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -13,5 +13,5 @@ class TestWebMaster(mastertest.MasterTest): def test_basic(self): m = self.mkmaster(None) for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/tox.ini b/tox.ini index bbfa438f..a7b5e7d3 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From e6e839d56d86e7f7126b3b662a07f12625f3d691 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 17:27:20 -0700 Subject: add python3 tnetstring implementation --- mitmproxy/contrib/py2/__init__.py | 0 mitmproxy/contrib/py2/tnetstring.py | 375 +++++++++++++++++++++++++++++ mitmproxy/contrib/py3/__init__.py | 0 mitmproxy/contrib/py3/tnetstring.py | 233 ++++++++++++++++++ mitmproxy/contrib/py3/tnetstring_tests.py | 133 +++++++++++ mitmproxy/contrib/tnetstring.py | 377 +----------------------------- 6 files changed, 746 insertions(+), 372 deletions(-) create mode 100644 mitmproxy/contrib/py2/__init__.py create mode 100644 mitmproxy/contrib/py2/tnetstring.py create mode 100644 mitmproxy/contrib/py3/__init__.py create mode 100644 mitmproxy/contrib/py3/tnetstring.py create mode 100644 mitmproxy/contrib/py3/tnetstring_tests.py diff --git a/mitmproxy/contrib/py2/__init__.py b/mitmproxy/contrib/py2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mitmproxy/contrib/py2/tnetstring.py b/mitmproxy/contrib/py2/tnetstring.py new file mode 100644 index 00000000..9bf20b09 --- /dev/null +++ b/mitmproxy/contrib/py2/tnetstring.py @@ -0,0 +1,375 @@ +# imported from the tnetstring project: https://github.com/rfk/tnetstring +# +# Copyright (c) 2011 Ryan Kelly +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +""" +tnetstring: data serialization using typed netstrings +====================================================== + + +This is a data serialization library. It's a lot like JSON but it uses a +new syntax called "typed netstrings" that Zed has proposed for use in the +Mongrel2 webserver. It's designed to be simpler and easier to implement +than JSON, with a happy consequence of also being faster in many cases. + +An ordinary netstring is a blob of data prefixed with its length and postfixed +with a sanity-checking comma. The string "hello world" encodes like this:: + + 11:hello world, + +Typed netstrings add other datatypes by replacing the comma with a type tag. +Here's the integer 12345 encoded as a tnetstring:: + + 5:12345# + +And here's the list [12345,True,0] which mixes integers and bools:: + + 19:5:12345#4:true!1:0#] + +Simple enough? This module gives you the following functions: + + :dump: dump an object as a tnetstring to a file + :dumps: dump an object as a tnetstring to a string + :load: load a tnetstring-encoded object from a file + :loads: load a tnetstring-encoded object from a string + :pop: pop a tnetstring-encoded object from the front of a string + +Note that since parsing a tnetstring requires reading all the data into memory +at once, there's no efficiency gain from using the file-based versions of these +functions. They're only here so you can use load() to read precisely one +item from a file or socket without consuming any extra data. + +By default tnetstrings work only with byte strings, not unicode. If you want +unicode strings then pass an optional encoding to the various functions, +like so:: + + >>> print(repr(tnetstring.loads("2:\\xce\\xb1,"))) + '\\xce\\xb1' + >>> + >>> print(repr(tnetstring.loads("2:\\xce\\xb1,","utf8"))) + u'\u03b1' + +""" +from collections import deque + +import six + +__ver_major__ = 0 +__ver_minor__ = 2 +__ver_patch__ = 0 +__ver_sub__ = "" +__version__ = "%d.%d.%d%s" % ( + __ver_major__, __ver_minor__, __ver_patch__, __ver_sub__) + + +def dumps(value): + """ + This function dumps a python object as a tnetstring. + """ + # This uses a deque to collect output fragments in reverse order, + # then joins them together at the end. It's measurably faster + # than creating all the intermediate strings. + # If you're reading this to get a handle on the tnetstring format, + # consider the _gdumps() function instead; it's a standard top-down + # generator that's simpler to understand but much less efficient. + q = deque() + _rdumpq(q, 0, value) + return b''.join(q) + + +def dump(value, file_handle): + """ + This function dumps a python object as a tnetstring and + writes it to the given file. + """ + file_handle.write(dumps(value)) + + +def _rdumpq(q, size, value): + """ + Dump value as a tnetstring, to a deque instance, last chunks first. + + This function generates the tnetstring representation of the given value, + pushing chunks of the output onto the given deque instance. It pushes + the last chunk first, then recursively generates more chunks. + + When passed in the current size of the string in the queue, it will return + the new size of the string in the queue. + + Operating last-chunk-first makes it easy to calculate the size written + for recursive structures without having to build their representation as + a string. This is measurably faster than generating the intermediate + strings, especially on deeply nested structures. + """ + write = q.appendleft + if value is None: + write(b'0:~') + return size + 3 + elif value is True: + write(b'4:true!') + return size + 7 + elif value is False: + write(b'5:false!') + return size + 8 + elif isinstance(value, six.integer_types): + data = str(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'#') + write(data) + write(b':') + write(span) + return size + 2 + len(span) + ldata + elif isinstance(value, float): + # Use repr() for float rather than str(). + # It round-trips more accurately. + # Probably unnecessary in later python versions that + # use David Gay's ftoa routines. + data = repr(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'^') + write(data) + write(b':') + write(span) + return size + 2 + len(span) + ldata + elif isinstance(value, bytes): + lvalue = len(value) + span = str(lvalue).encode() + write(b',') + write(value) + write(b':') + write(span) + return size + 2 + len(span) + lvalue + elif isinstance(value, (list, tuple)): + write(b']') + init_size = size = size + 1 + for item in reversed(value): + size = _rdumpq(q, size, item) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + elif isinstance(value, dict): + write(b'}') + init_size = size = size + 1 + for (k, v) in value.items(): + size = _rdumpq(q, size, v) + size = _rdumpq(q, size, k) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + else: + raise ValueError("unserializable object: {} ({})".format(value, type(value))) + + +def _gdumps(value): + """ + Generate fragments of value dumped as a tnetstring. + + This is the naive dumping algorithm, implemented as a generator so that + it's easy to pass to "".join() without building a new list. + + This is mainly here for comparison purposes; the _rdumpq version is + measurably faster as it doesn't have to build intermediate strins. + """ + if value is None: + yield b'0:~' + elif value is True: + yield b'4:true!' + elif value is False: + yield b'5:false!' + elif isinstance(value, six.integer_types): + data = str(value).encode() + yield str(len(data)).encode() + yield b':' + yield data + yield b'#' + elif isinstance(value, float): + data = repr(value).encode() + yield str(len(data)).encode() + yield b':' + yield data + yield b'^' + elif isinstance(value, bytes): + yield str(len(value)).encode() + yield b':' + yield value + yield b',' + elif isinstance(value, (list, tuple)): + sub = [] + for item in value: + sub.extend(_gdumps(item)) + sub = b''.join(sub) + yield str(len(sub)).encode() + yield b':' + yield sub + yield b']' + elif isinstance(value, (dict,)): + sub = [] + for (k, v) in value.items(): + sub.extend(_gdumps(k)) + sub.extend(_gdumps(v)) + sub = b''.join(sub) + yield str(len(sub)).encode() + yield b':' + yield sub + yield b'}' + else: + raise ValueError("unserializable object") + + +def loads(string): + """ + This function parses a tnetstring into a python object. + """ + # No point duplicating effort here. In the C-extension version, + # loads() is measurably faster then pop() since it can avoid + # the overhead of building a second string. + return pop(string)[0] + + +def load(file_handle): + """load(file) -> object + + This function reads a tnetstring from a file and parses it into a + python object. The file must support the read() method, and this + function promises not to read more data than necessary. + """ + # Read the length prefix one char at a time. + # Note that the netstring spec explicitly forbids padding zeros. + c = file_handle.read(1) + if not c.isdigit(): + raise ValueError("not a tnetstring: missing or invalid length prefix") + datalen = ord(c) - ord('0') + c = file_handle.read(1) + if datalen != 0: + while c.isdigit(): + datalen = (10 * datalen) + (ord(c) - ord('0')) + if datalen > 999999999: + errmsg = "not a tnetstring: absurdly large length prefix" + raise ValueError(errmsg) + c = file_handle.read(1) + if c != b':': + raise ValueError("not a tnetstring: missing or invalid length prefix") + # Now we can read and parse the payload. + # This repeats the dispatch logic of pop() so we can avoid + # re-constructing the outermost tnetstring. + data = file_handle.read(datalen) + if len(data) != datalen: + raise ValueError("not a tnetstring: length prefix too big") + tns_type = file_handle.read(1) + if tns_type == b',': + return data + if tns_type == b'#': + try: + return int(data) + except ValueError: + raise ValueError("not a tnetstring: invalid integer literal") + if tns_type == b'^': + try: + return float(data) + except ValueError: + raise ValueError("not a tnetstring: invalid float literal") + if tns_type == b'!': + if data == b'true': + return True + elif data == b'false': + return False + else: + raise ValueError("not a tnetstring: invalid boolean literal") + if tns_type == b'~': + if data: + raise ValueError("not a tnetstring: invalid null literal") + return None + if tns_type == b']': + l = [] + while data: + item, data = pop(data) + l.append(item) + return l + if tns_type == b'}': + d = {} + while data: + key, data = pop(data) + val, data = pop(data) + d[key] = val + return d + raise ValueError("unknown type tag") + + +def pop(string): + """pop(string,encoding='utf_8') -> (object, remain) + + This function parses a tnetstring into a python object. + It returns a tuple giving the parsed object and a string + containing any unparsed data from the end of the string. + """ + # Parse out data length, type and remaining string. + try: + dlen, rest = string.split(b':', 1) + dlen = int(dlen) + except ValueError: + raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(string)) + try: + data, tns_type, remain = rest[:dlen], rest[dlen:dlen + 1], rest[dlen + 1:] + except IndexError: + # This fires if len(rest) < dlen, meaning we don't need + # to further validate that data is the right length. + raise ValueError("not a tnetstring: invalid length prefix: {}".format(dlen)) + # Parse the data based on the type tag. + if tns_type == b',': + return data, remain + if tns_type == b'#': + try: + return int(data), remain + except ValueError: + raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) + if tns_type == b'^': + try: + return float(data), remain + except ValueError: + raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) + if tns_type == b'!': + if data == b'true': + return True, remain + elif data == b'false': + return False, remain + else: + raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) + if tns_type == b'~': + if data: + raise ValueError("not a tnetstring: invalid null literal") + return None, remain + if tns_type == b']': + l = [] + while data: + item, data = pop(data) + l.append(item) + return (l, remain) + if tns_type == b'}': + d = {} + while data: + key, data = pop(data) + val, data = pop(data) + d[key] = val + return d, remain + raise ValueError("unknown type tag: {}".format(tns_type)) diff --git a/mitmproxy/contrib/py3/__init__.py b/mitmproxy/contrib/py3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mitmproxy/contrib/py3/tnetstring.py b/mitmproxy/contrib/py3/tnetstring.py new file mode 100644 index 00000000..6f38a245 --- /dev/null +++ b/mitmproxy/contrib/py3/tnetstring.py @@ -0,0 +1,233 @@ +""" +tnetstring: data serialization using typed netstrings +====================================================== + +This is a custom Python 3 implementation of tnetstrings. +Compared to other implementations, the main difference +is the conversion of dictionary keys to str. + +An ordinary tnetstring is a blob of data prefixed with its length and postfixed +with its type. Here are some examples: + + >>> tnetstring.dumps("hello world") + 11:hello world, + >>> tnetstring.dumps(12345) + 5:12345# + >>> tnetstring.dumps([12345, True, 0]) + 19:5:12345#4:true!1:0#] + +This module gives you the following functions: + + :dump: dump an object as a tnetstring to a file + :dumps: dump an object as a tnetstring to a string + :load: load a tnetstring-encoded object from a file + :loads: load a tnetstring-encoded object from a string + +Note that since parsing a tnetstring requires reading all the data into memory +at once, there's no efficiency gain from using the file-based versions of these +functions. They're only here so you can use load() to read precisely one +item from a file or socket without consuming any extra data. + +The tnetstrings specification explicitly states that strings are binary blobs +and forbids the use of unicode at the protocol level. +**This implementation decodes dictionary keys as surrogate-escaped ASCII**, +all other strings are returned as plain bytes. + +:Copyright: (c) 2012-2013 by Ryan Kelly . +:Copyright: (c) 2014 by Carlo Pires . +:Copyright: (c) 2016 by Maximilian Hils . + +:License: MIT +""" + +import collections +from typing import io, Union, Tuple + +TSerializable = Union[None, bool, int, float, bytes, list, tuple, dict] + + +def dumps(value: TSerializable) -> bytes: + """ + This function dumps a python object as a tnetstring. + """ + # This uses a deque to collect output fragments in reverse order, + # then joins them together at the end. It's measurably faster + # than creating all the intermediate strings. + q = collections.deque() + _rdumpq(q, 0, value) + return b''.join(q) + + +def dump(value: TSerializable, file_handle: io.BinaryIO) -> None: + """ + This function dumps a python object as a tnetstring and + writes it to the given file. + """ + file_handle.write(dumps(value)) + + +def _rdumpq(q: collections.deque, size: int, value: TSerializable) -> int: + """ + Dump value as a tnetstring, to a deque instance, last chunks first. + + This function generates the tnetstring representation of the given value, + pushing chunks of the output onto the given deque instance. It pushes + the last chunk first, then recursively generates more chunks. + + When passed in the current size of the string in the queue, it will return + the new size of the string in the queue. + + Operating last-chunk-first makes it easy to calculate the size written + for recursive structures without having to build their representation as + a string. This is measurably faster than generating the intermediate + strings, especially on deeply nested structures. + """ + write = q.appendleft + if value is None: + write(b'0:~') + return size + 3 + elif value is True: + write(b'4:true!') + return size + 7 + elif value is False: + write(b'5:false!') + return size + 8 + elif isinstance(value, int): + data = str(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s#' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, float): + # Use repr() for float rather than str(). + # It round-trips more accurately. + # Probably unnecessary in later python versions that + # use David Gay's ftoa routines. + data = repr(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s^' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, bytes): + lvalue = len(value) + span = str(lvalue).encode() + write(b'%s:%s,' % (span, value)) + return size + 2 + len(span) + lvalue + elif isinstance(value, (list, tuple)): + write(b']') + init_size = size = size + 1 + for item in reversed(value): + size = _rdumpq(q, size, item) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + elif isinstance(value, dict): + write(b'}') + init_size = size = size + 1 + for (k, v) in value.items(): + size = _rdumpq(q, size, v) + size = _rdumpq(q, size, k) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + else: + raise ValueError("unserializable object: {} ({})".format(value, type(value))) + + +def loads(string: bytes) -> TSerializable: + """ + This function parses a tnetstring into a python object. + """ + return pop(string)[0] + + +def load(file_handle: io.BinaryIO) -> TSerializable: + """load(file) -> object + + This function reads a tnetstring from a file and parses it into a + python object. The file must support the read() method, and this + function promises not to read more data than necessary. + """ + # Read the length prefix one char at a time. + # Note that the netstring spec explicitly forbids padding zeros. + c = file_handle.read(1) + data_length = b"" + while ord(b'0') <= ord(c) <= ord(b'9'): + data_length += c + if len(data_length) > 9: + raise ValueError("not a tnetstring: absurdly large length prefix") + c = file_handle.read(1) + if c != b":": + raise ValueError("not a tnetstring: missing or invalid length prefix") + + data = file_handle.read(int(data_length)) + data_type = file_handle.read(1)[0] + + return parse(data_type, data) + + +def parse(data_type: int, data: bytes) -> TSerializable: + if data_type == ord(b','): + return data + if data_type == ord(b'#'): + try: + return int(data) + except ValueError: + raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) + if data_type == ord(b'^'): + try: + return float(data) + except ValueError: + raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) + if data_type == ord(b'!'): + if data == b'true': + return True + elif data == b'false': + return False + else: + raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) + if data_type == ord(b'~'): + if data: + raise ValueError("not a tnetstring: invalid null literal") + return None + if data_type == ord(b']'): + l = [] + while data: + item, data = pop(data) + l.append(item) + return l + if data_type == ord(b'}'): + d = {} + while data: + key, data = pop(data) + val, data = pop(data) + d[key] = val + return d + raise ValueError("unknown type tag: {}".format(data_type)) + + +def pop(data: bytes) -> Tuple[TSerializable, bytes]: + """ + This function parses a tnetstring into a python object. + It returns a tuple giving the parsed object and a string + containing any unparsed data from the end of the string. + """ + # Parse out data length, type and remaining string. + try: + length, data = data.split(b':', 1) + length = int(length) + except ValueError: + raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data)) + try: + data, data_type, remain = data[:length], data[length], data[length + 1:] + except IndexError: + # This fires if len(data) < dlen, meaning we don't need + # to further validate that data is the right length. + raise ValueError("not a tnetstring: invalid length prefix: {}".format(length)) + # Parse the data based on the type tag. + return parse(data_type, data), remain + + +__all__ = ["dump", "dumps", "load", "loads"] diff --git a/mitmproxy/contrib/py3/tnetstring_tests.py b/mitmproxy/contrib/py3/tnetstring_tests.py new file mode 100644 index 00000000..545889c8 --- /dev/null +++ b/mitmproxy/contrib/py3/tnetstring_tests.py @@ -0,0 +1,133 @@ +import unittest +import random +import math +import io +from . import tnetstring +import struct + +MAXINT = 2 ** (struct.Struct('i').size * 8 - 1) - 1 + +FORMAT_EXAMPLES = { + b'0:}': {}, + b'0:]': [], + b'51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00\x00,]}': + {b'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, + b'5:12345#': 12345, + b'12:this is cool,': b'this is cool', + b'0:,': b'', + b'0:~': None, + b'4:true!': True, + b'5:false!': False, + b'10:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,': b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', + b'24:5:12345#5:67890#5:xxxxx,]': [12345, 67890, b'xxxxx'], + b'18:3:0.1^3:0.2^3:0.3^]': [0.1, 0.2, 0.3], + b'243:238:233:228:223:218:213:208:203:198:193:188:183:178:173:168:163:158:153:148:143:138:133:128:123:118:113:108:103:99:95:91:87:83:79:75:71:67:63:59:55:51:47:43:39:35:31:27:23:19:15:11:hello-there,]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]': [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[b'hello-there']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +} + +def get_random_object(random=random, depth=0): + """Generate a random serializable object.""" + # The probability of generating a scalar value increases as the depth increase. + # This ensures that we bottom out eventually. + if random.randint(depth,10) <= 4: + what = random.randint(0,1) + if what == 0: + n = random.randint(0,10) + l = [] + for _ in range(n): + l.append(get_random_object(random,depth+1)) + return l + if what == 1: + n = random.randint(0,10) + d = {} + for _ in range(n): + n = random.randint(0,100) + k = bytes([random.randint(32,126) for _ in range(n)]) + d[k] = get_random_object(random,depth+1) + return d + else: + what = random.randint(0,4) + if what == 0: + return None + if what == 1: + return True + if what == 2: + return False + if what == 3: + if random.randint(0,1) == 0: + return random.randint(0,MAXINT) + else: + return -1 * random.randint(0,MAXINT) + n = random.randint(0,100) + return bytes([random.randint(32,126) for _ in range(n)]) + +class Test_Format(unittest.TestCase): + def test_roundtrip_format_examples(self): + for data, expect in FORMAT_EXAMPLES.items(): + self.assertEqual(expect,tnetstring.loads(data)) + self.assertEqual(expect,tnetstring.loads(tnetstring.dumps(expect))) + self.assertEqual((expect,b''),tnetstring.pop(data)) + + def test_roundtrip_format_random(self): + for _ in range(500): + v = get_random_object() + self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v))) + self.assertEqual((v,b""),tnetstring.pop(tnetstring.dumps(v))) + + def test_unicode_handling(self): + with self.assertRaises(ValueError): + tnetstring.dumps("hello") + self.assertEqual(tnetstring.dumps("hello".encode()),b"5:hello,") + self.assertEqual(type(tnetstring.loads(b"5:hello,")),bytes) + + def test_roundtrip_format_unicode(self): + for _ in range(500): + v = get_random_object() + self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v))) + self.assertEqual((v,b''),tnetstring.pop(tnetstring.dumps(v))) + + def test_roundtrip_big_integer(self): + i1 = math.factorial(30000) + s = tnetstring.dumps(i1) + i2 = tnetstring.loads(s) + self.assertEqual(i1, i2) + +class Test_FileLoading(unittest.TestCase): + def test_roundtrip_file_examples(self): + for data, expect in FORMAT_EXAMPLES.items(): + s = io.BytesIO() + s.write(data) + s.write(b'OK') + s.seek(0) + self.assertEqual(expect,tnetstring.load(s)) + self.assertEqual(b'OK',s.read()) + s = io.BytesIO() + tnetstring.dump(expect,s) + s.write(b'OK') + s.seek(0) + self.assertEqual(expect,tnetstring.load(s)) + self.assertEqual(b'OK',s.read()) + + def test_roundtrip_file_random(self): + for _ in range(500): + v = get_random_object() + s = io.BytesIO() + tnetstring.dump(v,s) + s.write(b'OK') + s.seek(0) + self.assertEqual(v,tnetstring.load(s)) + self.assertEqual(b'OK',s.read()) + + def test_error_on_absurd_lengths(self): + s = io.BytesIO() + s.write(b'1000000000:pwned!,') + s.seek(0) + with self.assertRaises(ValueError): + tnetstring.load(s) + self.assertEqual(s.read(1),b':') + +def suite(): + loader = unittest.TestLoader() + suite = unittest.TestSuite() + suite.addTest(loader.loadTestsFromTestCase(Test_Format)) + suite.addTest(loader.loadTestsFromTestCase(Test_FileLoading)) + return suite \ No newline at end of file diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index 9bf20b09..58daec5c 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -1,375 +1,8 @@ -# imported from the tnetstring project: https://github.com/rfk/tnetstring -# -# Copyright (c) 2011 Ryan Kelly -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -""" -tnetstring: data serialization using typed netstrings -====================================================== - - -This is a data serialization library. It's a lot like JSON but it uses a -new syntax called "typed netstrings" that Zed has proposed for use in the -Mongrel2 webserver. It's designed to be simpler and easier to implement -than JSON, with a happy consequence of also being faster in many cases. - -An ordinary netstring is a blob of data prefixed with its length and postfixed -with a sanity-checking comma. The string "hello world" encodes like this:: - - 11:hello world, - -Typed netstrings add other datatypes by replacing the comma with a type tag. -Here's the integer 12345 encoded as a tnetstring:: - - 5:12345# - -And here's the list [12345,True,0] which mixes integers and bools:: - - 19:5:12345#4:true!1:0#] - -Simple enough? This module gives you the following functions: - - :dump: dump an object as a tnetstring to a file - :dumps: dump an object as a tnetstring to a string - :load: load a tnetstring-encoded object from a file - :loads: load a tnetstring-encoded object from a string - :pop: pop a tnetstring-encoded object from the front of a string - -Note that since parsing a tnetstring requires reading all the data into memory -at once, there's no efficiency gain from using the file-based versions of these -functions. They're only here so you can use load() to read precisely one -item from a file or socket without consuming any extra data. - -By default tnetstrings work only with byte strings, not unicode. If you want -unicode strings then pass an optional encoding to the various functions, -like so:: - - >>> print(repr(tnetstring.loads("2:\\xce\\xb1,"))) - '\\xce\\xb1' - >>> - >>> print(repr(tnetstring.loads("2:\\xce\\xb1,","utf8"))) - u'\u03b1' - -""" -from collections import deque - import six -__ver_major__ = 0 -__ver_minor__ = 2 -__ver_patch__ = 0 -__ver_sub__ = "" -__version__ = "%d.%d.%d%s" % ( - __ver_major__, __ver_minor__, __ver_patch__, __ver_sub__) - - -def dumps(value): - """ - This function dumps a python object as a tnetstring. - """ - # This uses a deque to collect output fragments in reverse order, - # then joins them together at the end. It's measurably faster - # than creating all the intermediate strings. - # If you're reading this to get a handle on the tnetstring format, - # consider the _gdumps() function instead; it's a standard top-down - # generator that's simpler to understand but much less efficient. - q = deque() - _rdumpq(q, 0, value) - return b''.join(q) - - -def dump(value, file_handle): - """ - This function dumps a python object as a tnetstring and - writes it to the given file. - """ - file_handle.write(dumps(value)) - - -def _rdumpq(q, size, value): - """ - Dump value as a tnetstring, to a deque instance, last chunks first. - - This function generates the tnetstring representation of the given value, - pushing chunks of the output onto the given deque instance. It pushes - the last chunk first, then recursively generates more chunks. - - When passed in the current size of the string in the queue, it will return - the new size of the string in the queue. - - Operating last-chunk-first makes it easy to calculate the size written - for recursive structures without having to build their representation as - a string. This is measurably faster than generating the intermediate - strings, especially on deeply nested structures. - """ - write = q.appendleft - if value is None: - write(b'0:~') - return size + 3 - elif value is True: - write(b'4:true!') - return size + 7 - elif value is False: - write(b'5:false!') - return size + 8 - elif isinstance(value, six.integer_types): - data = str(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'#') - write(data) - write(b':') - write(span) - return size + 2 + len(span) + ldata - elif isinstance(value, float): - # Use repr() for float rather than str(). - # It round-trips more accurately. - # Probably unnecessary in later python versions that - # use David Gay's ftoa routines. - data = repr(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'^') - write(data) - write(b':') - write(span) - return size + 2 + len(span) + ldata - elif isinstance(value, bytes): - lvalue = len(value) - span = str(lvalue).encode() - write(b',') - write(value) - write(b':') - write(span) - return size + 2 + len(span) + lvalue - elif isinstance(value, (list, tuple)): - write(b']') - init_size = size = size + 1 - for item in reversed(value): - size = _rdumpq(q, size, item) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - elif isinstance(value, dict): - write(b'}') - init_size = size = size + 1 - for (k, v) in value.items(): - size = _rdumpq(q, size, v) - size = _rdumpq(q, size, k) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - else: - raise ValueError("unserializable object: {} ({})".format(value, type(value))) - - -def _gdumps(value): - """ - Generate fragments of value dumped as a tnetstring. - - This is the naive dumping algorithm, implemented as a generator so that - it's easy to pass to "".join() without building a new list. - - This is mainly here for comparison purposes; the _rdumpq version is - measurably faster as it doesn't have to build intermediate strins. - """ - if value is None: - yield b'0:~' - elif value is True: - yield b'4:true!' - elif value is False: - yield b'5:false!' - elif isinstance(value, six.integer_types): - data = str(value).encode() - yield str(len(data)).encode() - yield b':' - yield data - yield b'#' - elif isinstance(value, float): - data = repr(value).encode() - yield str(len(data)).encode() - yield b':' - yield data - yield b'^' - elif isinstance(value, bytes): - yield str(len(value)).encode() - yield b':' - yield value - yield b',' - elif isinstance(value, (list, tuple)): - sub = [] - for item in value: - sub.extend(_gdumps(item)) - sub = b''.join(sub) - yield str(len(sub)).encode() - yield b':' - yield sub - yield b']' - elif isinstance(value, (dict,)): - sub = [] - for (k, v) in value.items(): - sub.extend(_gdumps(k)) - sub.extend(_gdumps(v)) - sub = b''.join(sub) - yield str(len(sub)).encode() - yield b':' - yield sub - yield b'}' - else: - raise ValueError("unserializable object") - - -def loads(string): - """ - This function parses a tnetstring into a python object. - """ - # No point duplicating effort here. In the C-extension version, - # loads() is measurably faster then pop() since it can avoid - # the overhead of building a second string. - return pop(string)[0] - - -def load(file_handle): - """load(file) -> object - - This function reads a tnetstring from a file and parses it into a - python object. The file must support the read() method, and this - function promises not to read more data than necessary. - """ - # Read the length prefix one char at a time. - # Note that the netstring spec explicitly forbids padding zeros. - c = file_handle.read(1) - if not c.isdigit(): - raise ValueError("not a tnetstring: missing or invalid length prefix") - datalen = ord(c) - ord('0') - c = file_handle.read(1) - if datalen != 0: - while c.isdigit(): - datalen = (10 * datalen) + (ord(c) - ord('0')) - if datalen > 999999999: - errmsg = "not a tnetstring: absurdly large length prefix" - raise ValueError(errmsg) - c = file_handle.read(1) - if c != b':': - raise ValueError("not a tnetstring: missing or invalid length prefix") - # Now we can read and parse the payload. - # This repeats the dispatch logic of pop() so we can avoid - # re-constructing the outermost tnetstring. - data = file_handle.read(datalen) - if len(data) != datalen: - raise ValueError("not a tnetstring: length prefix too big") - tns_type = file_handle.read(1) - if tns_type == b',': - return data - if tns_type == b'#': - try: - return int(data) - except ValueError: - raise ValueError("not a tnetstring: invalid integer literal") - if tns_type == b'^': - try: - return float(data) - except ValueError: - raise ValueError("not a tnetstring: invalid float literal") - if tns_type == b'!': - if data == b'true': - return True - elif data == b'false': - return False - else: - raise ValueError("not a tnetstring: invalid boolean literal") - if tns_type == b'~': - if data: - raise ValueError("not a tnetstring: invalid null literal") - return None - if tns_type == b']': - l = [] - while data: - item, data = pop(data) - l.append(item) - return l - if tns_type == b'}': - d = {} - while data: - key, data = pop(data) - val, data = pop(data) - d[key] = val - return d - raise ValueError("unknown type tag") - - -def pop(string): - """pop(string,encoding='utf_8') -> (object, remain) +if six.PY2: + from .py2.tnetstring import load, loads, dump, dumps +else: + from .py3.tnetstring import load, loads, dump, dumps - This function parses a tnetstring into a python object. - It returns a tuple giving the parsed object and a string - containing any unparsed data from the end of the string. - """ - # Parse out data length, type and remaining string. - try: - dlen, rest = string.split(b':', 1) - dlen = int(dlen) - except ValueError: - raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(string)) - try: - data, tns_type, remain = rest[:dlen], rest[dlen:dlen + 1], rest[dlen + 1:] - except IndexError: - # This fires if len(rest) < dlen, meaning we don't need - # to further validate that data is the right length. - raise ValueError("not a tnetstring: invalid length prefix: {}".format(dlen)) - # Parse the data based on the type tag. - if tns_type == b',': - return data, remain - if tns_type == b'#': - try: - return int(data), remain - except ValueError: - raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) - if tns_type == b'^': - try: - return float(data), remain - except ValueError: - raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) - if tns_type == b'!': - if data == b'true': - return True, remain - elif data == b'false': - return False, remain - else: - raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) - if tns_type == b'~': - if data: - raise ValueError("not a tnetstring: invalid null literal") - return None, remain - if tns_type == b']': - l = [] - while data: - item, data = pop(data) - l.append(item) - return (l, remain) - if tns_type == b'}': - d = {} - while data: - key, data = pop(data) - val, data = pop(data) - d[key] = val - return d, remain - raise ValueError("unknown type tag: {}".format(tns_type)) +__all__ = ["load", "loads", "dump", "dumps"] -- cgit v1.2.3 From 684b4b5130aa9cc75322dd270172b263615d39dc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 18:48:45 -0700 Subject: tnetstring: keys are str on py3. migrate flow.io_compat --- mitmproxy/contrib/py3/tnetstring.py | 8 ++++++-- mitmproxy/contrib/py3/tnetstring_tests.py | 4 ++-- mitmproxy/contrib/tnetstring.py | 6 +++--- mitmproxy/flow/io.py | 9 +++++---- mitmproxy/flow/io_compat.py | 3 +-- test/mitmproxy/test_contrib_tnetstring.py | 4 ++-- tox.ini | 2 +- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/mitmproxy/contrib/py3/tnetstring.py b/mitmproxy/contrib/py3/tnetstring.py index 6f38a245..6998fc82 100644 --- a/mitmproxy/contrib/py3/tnetstring.py +++ b/mitmproxy/contrib/py3/tnetstring.py @@ -126,6 +126,8 @@ def _rdumpq(q: collections.deque, size: int, value: TSerializable) -> int: write(b'}') init_size = size = size + 1 for (k, v) in value.items(): + if isinstance(k, str): + k = k.encode("ascii", "strict") size = _rdumpq(q, size, v) size = _rdumpq(q, size, k) span = str(size - init_size).encode() @@ -154,7 +156,7 @@ def load(file_handle: io.BinaryIO) -> TSerializable: # Note that the netstring spec explicitly forbids padding zeros. c = file_handle.read(1) data_length = b"" - while ord(b'0') <= ord(c) <= ord(b'9'): + while c.isdigit(): data_length += c if len(data_length) > 9: raise ValueError("not a tnetstring: absurdly large length prefix") @@ -202,6 +204,8 @@ def parse(data_type: int, data: bytes) -> TSerializable: d = {} while data: key, data = pop(data) + if isinstance(key, bytes): + key = key.decode("ascii", "strict") val, data = pop(data) d[key] = val return d @@ -230,4 +234,4 @@ def pop(data: bytes) -> Tuple[TSerializable, bytes]: return parse(data_type, data), remain -__all__ = ["dump", "dumps", "load", "loads"] +__all__ = ["dump", "dumps", "load", "loads", "pop"] diff --git a/mitmproxy/contrib/py3/tnetstring_tests.py b/mitmproxy/contrib/py3/tnetstring_tests.py index 545889c8..4ee184d5 100644 --- a/mitmproxy/contrib/py3/tnetstring_tests.py +++ b/mitmproxy/contrib/py3/tnetstring_tests.py @@ -11,7 +11,7 @@ FORMAT_EXAMPLES = { b'0:}': {}, b'0:]': [], b'51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00\x00,]}': - {b'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, + {'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, b'5:12345#': 12345, b'12:this is cool,': b'this is cool', b'0:,': b'', @@ -41,7 +41,7 @@ def get_random_object(random=random, depth=0): d = {} for _ in range(n): n = random.randint(0,100) - k = bytes([random.randint(32,126) for _ in range(n)]) + k = str([random.randint(32,126) for _ in range(n)]) d[k] = get_random_object(random,depth+1) return d else: diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index 58daec5c..1ebaba21 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -1,8 +1,8 @@ import six if six.PY2: - from .py2.tnetstring import load, loads, dump, dumps + from .py2.tnetstring import load, loads, dump, dumps, pop else: - from .py3.tnetstring import load, loads, dump, dumps + from .py3.tnetstring import load, loads, dump, dumps, pop -__all__ = ["load", "loads", "dump", "dumps"] +__all__ = ["load", "loads", "dump", "dumps", "pop"] diff --git a/mitmproxy/flow/io.py b/mitmproxy/flow/io.py index 671ddf43..e5716940 100644 --- a/mitmproxy/flow/io.py +++ b/mitmproxy/flow/io.py @@ -44,12 +44,13 @@ class FlowReader: raise exceptions.FlowReadException(str(e)) if can_tell: off = self.fo.tell() - if data["type"] not in models.FLOW_TYPES: - raise exceptions.FlowReadException("Unknown flow type: {}".format(data["type"])) - yield models.FLOW_TYPES[data["type"]].from_state(data) + data_type = data["type"].decode() + if data_type not in models.FLOW_TYPES: + raise exceptions.FlowReadException("Unknown flow type: {}".format(data_type)) + yield models.FLOW_TYPES[data_type].from_state(data) except ValueError: # Error is due to EOF - if can_tell and self.fo.tell() == off and self.fo.read() == '': + if can_tell and self.fo.tell() == off and self.fo.read() == b'': return raise exceptions.FlowReadException("Invalid data format.") diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index 1023e87f..55971f5e 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -9,6 +9,7 @@ from netlib import version def convert_013_014(data): data["request"]["first_line_format"] = data["request"].pop("form_in") data["request"]["http_version"] = "HTTP/" + ".".join(str(x) for x in data["request"].pop("httpversion")) + data["response"]["http_version"] = "HTTP/" + ".".join(str(x) for x in data["response"].pop("httpversion")) data["response"]["status_code"] = data["response"].pop("code") data["response"]["body"] = data["response"].pop("content") data["server_conn"].pop("state") @@ -26,8 +27,6 @@ def convert_015_016(data): for m in ("request", "response"): if "body" in data[m]: data[m]["content"] = data[m].pop("body") - if "httpversion" in data[m]: - data[m]["http_version"] = data[m].pop("httpversion") if "msg" in data["response"]: data["response"]["reason"] = data["response"].pop("msg") data["request"].pop("form_out", None) diff --git a/test/mitmproxy/test_contrib_tnetstring.py b/test/mitmproxy/test_contrib_tnetstring.py index 17654ad9..8ae35a25 100644 --- a/test/mitmproxy/test_contrib_tnetstring.py +++ b/test/mitmproxy/test_contrib_tnetstring.py @@ -12,7 +12,7 @@ FORMAT_EXAMPLES = { b'0:}': {}, b'0:]': [], b'51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00\x00,]}': - {b'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, + {'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, b'5:12345#': 12345, b'12:this is cool,': b'this is cool', b'0:,': b'', @@ -43,7 +43,7 @@ def get_random_object(random=random, depth=0): d = {} for _ in range(n): n = random.randint(0, 100) - k = bytes([random.randint(32, 126) for _ in range(n)]) + k = str([random.randint(32, 126) for _ in range(n)]) d[k] = get_random_object(random, depth + 1) return d else: diff --git a/tox.ini b/tox.ini index a7b5e7d3..251609a5 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From 48ee3a553e30b36c16bfbe1674d3313605dff661 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 19:25:56 -0700 Subject: add tnetstring unicode type --- mitmproxy/contrib/py2/__init__.py | 0 mitmproxy/contrib/py2/tnetstring.py | 375 ------------------------------ mitmproxy/contrib/py3/__init__.py | 0 mitmproxy/contrib/py3/tnetstring.py | 237 ------------------- mitmproxy/contrib/py3/tnetstring_tests.py | 133 ----------- mitmproxy/contrib/tnetstring.py | 256 +++++++++++++++++++- 6 files changed, 251 insertions(+), 750 deletions(-) delete mode 100644 mitmproxy/contrib/py2/__init__.py delete mode 100644 mitmproxy/contrib/py2/tnetstring.py delete mode 100644 mitmproxy/contrib/py3/__init__.py delete mode 100644 mitmproxy/contrib/py3/tnetstring.py delete mode 100644 mitmproxy/contrib/py3/tnetstring_tests.py diff --git a/mitmproxy/contrib/py2/__init__.py b/mitmproxy/contrib/py2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mitmproxy/contrib/py2/tnetstring.py b/mitmproxy/contrib/py2/tnetstring.py deleted file mode 100644 index 9bf20b09..00000000 --- a/mitmproxy/contrib/py2/tnetstring.py +++ /dev/null @@ -1,375 +0,0 @@ -# imported from the tnetstring project: https://github.com/rfk/tnetstring -# -# Copyright (c) 2011 Ryan Kelly -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -""" -tnetstring: data serialization using typed netstrings -====================================================== - - -This is a data serialization library. It's a lot like JSON but it uses a -new syntax called "typed netstrings" that Zed has proposed for use in the -Mongrel2 webserver. It's designed to be simpler and easier to implement -than JSON, with a happy consequence of also being faster in many cases. - -An ordinary netstring is a blob of data prefixed with its length and postfixed -with a sanity-checking comma. The string "hello world" encodes like this:: - - 11:hello world, - -Typed netstrings add other datatypes by replacing the comma with a type tag. -Here's the integer 12345 encoded as a tnetstring:: - - 5:12345# - -And here's the list [12345,True,0] which mixes integers and bools:: - - 19:5:12345#4:true!1:0#] - -Simple enough? This module gives you the following functions: - - :dump: dump an object as a tnetstring to a file - :dumps: dump an object as a tnetstring to a string - :load: load a tnetstring-encoded object from a file - :loads: load a tnetstring-encoded object from a string - :pop: pop a tnetstring-encoded object from the front of a string - -Note that since parsing a tnetstring requires reading all the data into memory -at once, there's no efficiency gain from using the file-based versions of these -functions. They're only here so you can use load() to read precisely one -item from a file or socket without consuming any extra data. - -By default tnetstrings work only with byte strings, not unicode. If you want -unicode strings then pass an optional encoding to the various functions, -like so:: - - >>> print(repr(tnetstring.loads("2:\\xce\\xb1,"))) - '\\xce\\xb1' - >>> - >>> print(repr(tnetstring.loads("2:\\xce\\xb1,","utf8"))) - u'\u03b1' - -""" -from collections import deque - -import six - -__ver_major__ = 0 -__ver_minor__ = 2 -__ver_patch__ = 0 -__ver_sub__ = "" -__version__ = "%d.%d.%d%s" % ( - __ver_major__, __ver_minor__, __ver_patch__, __ver_sub__) - - -def dumps(value): - """ - This function dumps a python object as a tnetstring. - """ - # This uses a deque to collect output fragments in reverse order, - # then joins them together at the end. It's measurably faster - # than creating all the intermediate strings. - # If you're reading this to get a handle on the tnetstring format, - # consider the _gdumps() function instead; it's a standard top-down - # generator that's simpler to understand but much less efficient. - q = deque() - _rdumpq(q, 0, value) - return b''.join(q) - - -def dump(value, file_handle): - """ - This function dumps a python object as a tnetstring and - writes it to the given file. - """ - file_handle.write(dumps(value)) - - -def _rdumpq(q, size, value): - """ - Dump value as a tnetstring, to a deque instance, last chunks first. - - This function generates the tnetstring representation of the given value, - pushing chunks of the output onto the given deque instance. It pushes - the last chunk first, then recursively generates more chunks. - - When passed in the current size of the string in the queue, it will return - the new size of the string in the queue. - - Operating last-chunk-first makes it easy to calculate the size written - for recursive structures without having to build their representation as - a string. This is measurably faster than generating the intermediate - strings, especially on deeply nested structures. - """ - write = q.appendleft - if value is None: - write(b'0:~') - return size + 3 - elif value is True: - write(b'4:true!') - return size + 7 - elif value is False: - write(b'5:false!') - return size + 8 - elif isinstance(value, six.integer_types): - data = str(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'#') - write(data) - write(b':') - write(span) - return size + 2 + len(span) + ldata - elif isinstance(value, float): - # Use repr() for float rather than str(). - # It round-trips more accurately. - # Probably unnecessary in later python versions that - # use David Gay's ftoa routines. - data = repr(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'^') - write(data) - write(b':') - write(span) - return size + 2 + len(span) + ldata - elif isinstance(value, bytes): - lvalue = len(value) - span = str(lvalue).encode() - write(b',') - write(value) - write(b':') - write(span) - return size + 2 + len(span) + lvalue - elif isinstance(value, (list, tuple)): - write(b']') - init_size = size = size + 1 - for item in reversed(value): - size = _rdumpq(q, size, item) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - elif isinstance(value, dict): - write(b'}') - init_size = size = size + 1 - for (k, v) in value.items(): - size = _rdumpq(q, size, v) - size = _rdumpq(q, size, k) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - else: - raise ValueError("unserializable object: {} ({})".format(value, type(value))) - - -def _gdumps(value): - """ - Generate fragments of value dumped as a tnetstring. - - This is the naive dumping algorithm, implemented as a generator so that - it's easy to pass to "".join() without building a new list. - - This is mainly here for comparison purposes; the _rdumpq version is - measurably faster as it doesn't have to build intermediate strins. - """ - if value is None: - yield b'0:~' - elif value is True: - yield b'4:true!' - elif value is False: - yield b'5:false!' - elif isinstance(value, six.integer_types): - data = str(value).encode() - yield str(len(data)).encode() - yield b':' - yield data - yield b'#' - elif isinstance(value, float): - data = repr(value).encode() - yield str(len(data)).encode() - yield b':' - yield data - yield b'^' - elif isinstance(value, bytes): - yield str(len(value)).encode() - yield b':' - yield value - yield b',' - elif isinstance(value, (list, tuple)): - sub = [] - for item in value: - sub.extend(_gdumps(item)) - sub = b''.join(sub) - yield str(len(sub)).encode() - yield b':' - yield sub - yield b']' - elif isinstance(value, (dict,)): - sub = [] - for (k, v) in value.items(): - sub.extend(_gdumps(k)) - sub.extend(_gdumps(v)) - sub = b''.join(sub) - yield str(len(sub)).encode() - yield b':' - yield sub - yield b'}' - else: - raise ValueError("unserializable object") - - -def loads(string): - """ - This function parses a tnetstring into a python object. - """ - # No point duplicating effort here. In the C-extension version, - # loads() is measurably faster then pop() since it can avoid - # the overhead of building a second string. - return pop(string)[0] - - -def load(file_handle): - """load(file) -> object - - This function reads a tnetstring from a file and parses it into a - python object. The file must support the read() method, and this - function promises not to read more data than necessary. - """ - # Read the length prefix one char at a time. - # Note that the netstring spec explicitly forbids padding zeros. - c = file_handle.read(1) - if not c.isdigit(): - raise ValueError("not a tnetstring: missing or invalid length prefix") - datalen = ord(c) - ord('0') - c = file_handle.read(1) - if datalen != 0: - while c.isdigit(): - datalen = (10 * datalen) + (ord(c) - ord('0')) - if datalen > 999999999: - errmsg = "not a tnetstring: absurdly large length prefix" - raise ValueError(errmsg) - c = file_handle.read(1) - if c != b':': - raise ValueError("not a tnetstring: missing or invalid length prefix") - # Now we can read and parse the payload. - # This repeats the dispatch logic of pop() so we can avoid - # re-constructing the outermost tnetstring. - data = file_handle.read(datalen) - if len(data) != datalen: - raise ValueError("not a tnetstring: length prefix too big") - tns_type = file_handle.read(1) - if tns_type == b',': - return data - if tns_type == b'#': - try: - return int(data) - except ValueError: - raise ValueError("not a tnetstring: invalid integer literal") - if tns_type == b'^': - try: - return float(data) - except ValueError: - raise ValueError("not a tnetstring: invalid float literal") - if tns_type == b'!': - if data == b'true': - return True - elif data == b'false': - return False - else: - raise ValueError("not a tnetstring: invalid boolean literal") - if tns_type == b'~': - if data: - raise ValueError("not a tnetstring: invalid null literal") - return None - if tns_type == b']': - l = [] - while data: - item, data = pop(data) - l.append(item) - return l - if tns_type == b'}': - d = {} - while data: - key, data = pop(data) - val, data = pop(data) - d[key] = val - return d - raise ValueError("unknown type tag") - - -def pop(string): - """pop(string,encoding='utf_8') -> (object, remain) - - This function parses a tnetstring into a python object. - It returns a tuple giving the parsed object and a string - containing any unparsed data from the end of the string. - """ - # Parse out data length, type and remaining string. - try: - dlen, rest = string.split(b':', 1) - dlen = int(dlen) - except ValueError: - raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(string)) - try: - data, tns_type, remain = rest[:dlen], rest[dlen:dlen + 1], rest[dlen + 1:] - except IndexError: - # This fires if len(rest) < dlen, meaning we don't need - # to further validate that data is the right length. - raise ValueError("not a tnetstring: invalid length prefix: {}".format(dlen)) - # Parse the data based on the type tag. - if tns_type == b',': - return data, remain - if tns_type == b'#': - try: - return int(data), remain - except ValueError: - raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) - if tns_type == b'^': - try: - return float(data), remain - except ValueError: - raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) - if tns_type == b'!': - if data == b'true': - return True, remain - elif data == b'false': - return False, remain - else: - raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) - if tns_type == b'~': - if data: - raise ValueError("not a tnetstring: invalid null literal") - return None, remain - if tns_type == b']': - l = [] - while data: - item, data = pop(data) - l.append(item) - return (l, remain) - if tns_type == b'}': - d = {} - while data: - key, data = pop(data) - val, data = pop(data) - d[key] = val - return d, remain - raise ValueError("unknown type tag: {}".format(tns_type)) diff --git a/mitmproxy/contrib/py3/__init__.py b/mitmproxy/contrib/py3/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mitmproxy/contrib/py3/tnetstring.py b/mitmproxy/contrib/py3/tnetstring.py deleted file mode 100644 index 6998fc82..00000000 --- a/mitmproxy/contrib/py3/tnetstring.py +++ /dev/null @@ -1,237 +0,0 @@ -""" -tnetstring: data serialization using typed netstrings -====================================================== - -This is a custom Python 3 implementation of tnetstrings. -Compared to other implementations, the main difference -is the conversion of dictionary keys to str. - -An ordinary tnetstring is a blob of data prefixed with its length and postfixed -with its type. Here are some examples: - - >>> tnetstring.dumps("hello world") - 11:hello world, - >>> tnetstring.dumps(12345) - 5:12345# - >>> tnetstring.dumps([12345, True, 0]) - 19:5:12345#4:true!1:0#] - -This module gives you the following functions: - - :dump: dump an object as a tnetstring to a file - :dumps: dump an object as a tnetstring to a string - :load: load a tnetstring-encoded object from a file - :loads: load a tnetstring-encoded object from a string - -Note that since parsing a tnetstring requires reading all the data into memory -at once, there's no efficiency gain from using the file-based versions of these -functions. They're only here so you can use load() to read precisely one -item from a file or socket without consuming any extra data. - -The tnetstrings specification explicitly states that strings are binary blobs -and forbids the use of unicode at the protocol level. -**This implementation decodes dictionary keys as surrogate-escaped ASCII**, -all other strings are returned as plain bytes. - -:Copyright: (c) 2012-2013 by Ryan Kelly . -:Copyright: (c) 2014 by Carlo Pires . -:Copyright: (c) 2016 by Maximilian Hils . - -:License: MIT -""" - -import collections -from typing import io, Union, Tuple - -TSerializable = Union[None, bool, int, float, bytes, list, tuple, dict] - - -def dumps(value: TSerializable) -> bytes: - """ - This function dumps a python object as a tnetstring. - """ - # This uses a deque to collect output fragments in reverse order, - # then joins them together at the end. It's measurably faster - # than creating all the intermediate strings. - q = collections.deque() - _rdumpq(q, 0, value) - return b''.join(q) - - -def dump(value: TSerializable, file_handle: io.BinaryIO) -> None: - """ - This function dumps a python object as a tnetstring and - writes it to the given file. - """ - file_handle.write(dumps(value)) - - -def _rdumpq(q: collections.deque, size: int, value: TSerializable) -> int: - """ - Dump value as a tnetstring, to a deque instance, last chunks first. - - This function generates the tnetstring representation of the given value, - pushing chunks of the output onto the given deque instance. It pushes - the last chunk first, then recursively generates more chunks. - - When passed in the current size of the string in the queue, it will return - the new size of the string in the queue. - - Operating last-chunk-first makes it easy to calculate the size written - for recursive structures without having to build their representation as - a string. This is measurably faster than generating the intermediate - strings, especially on deeply nested structures. - """ - write = q.appendleft - if value is None: - write(b'0:~') - return size + 3 - elif value is True: - write(b'4:true!') - return size + 7 - elif value is False: - write(b'5:false!') - return size + 8 - elif isinstance(value, int): - data = str(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'%s:%s#' % (span, data)) - return size + 2 + len(span) + ldata - elif isinstance(value, float): - # Use repr() for float rather than str(). - # It round-trips more accurately. - # Probably unnecessary in later python versions that - # use David Gay's ftoa routines. - data = repr(value).encode() - ldata = len(data) - span = str(ldata).encode() - write(b'%s:%s^' % (span, data)) - return size + 2 + len(span) + ldata - elif isinstance(value, bytes): - lvalue = len(value) - span = str(lvalue).encode() - write(b'%s:%s,' % (span, value)) - return size + 2 + len(span) + lvalue - elif isinstance(value, (list, tuple)): - write(b']') - init_size = size = size + 1 - for item in reversed(value): - size = _rdumpq(q, size, item) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - elif isinstance(value, dict): - write(b'}') - init_size = size = size + 1 - for (k, v) in value.items(): - if isinstance(k, str): - k = k.encode("ascii", "strict") - size = _rdumpq(q, size, v) - size = _rdumpq(q, size, k) - span = str(size - init_size).encode() - write(b':') - write(span) - return size + 1 + len(span) - else: - raise ValueError("unserializable object: {} ({})".format(value, type(value))) - - -def loads(string: bytes) -> TSerializable: - """ - This function parses a tnetstring into a python object. - """ - return pop(string)[0] - - -def load(file_handle: io.BinaryIO) -> TSerializable: - """load(file) -> object - - This function reads a tnetstring from a file and parses it into a - python object. The file must support the read() method, and this - function promises not to read more data than necessary. - """ - # Read the length prefix one char at a time. - # Note that the netstring spec explicitly forbids padding zeros. - c = file_handle.read(1) - data_length = b"" - while c.isdigit(): - data_length += c - if len(data_length) > 9: - raise ValueError("not a tnetstring: absurdly large length prefix") - c = file_handle.read(1) - if c != b":": - raise ValueError("not a tnetstring: missing or invalid length prefix") - - data = file_handle.read(int(data_length)) - data_type = file_handle.read(1)[0] - - return parse(data_type, data) - - -def parse(data_type: int, data: bytes) -> TSerializable: - if data_type == ord(b','): - return data - if data_type == ord(b'#'): - try: - return int(data) - except ValueError: - raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) - if data_type == ord(b'^'): - try: - return float(data) - except ValueError: - raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) - if data_type == ord(b'!'): - if data == b'true': - return True - elif data == b'false': - return False - else: - raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) - if data_type == ord(b'~'): - if data: - raise ValueError("not a tnetstring: invalid null literal") - return None - if data_type == ord(b']'): - l = [] - while data: - item, data = pop(data) - l.append(item) - return l - if data_type == ord(b'}'): - d = {} - while data: - key, data = pop(data) - if isinstance(key, bytes): - key = key.decode("ascii", "strict") - val, data = pop(data) - d[key] = val - return d - raise ValueError("unknown type tag: {}".format(data_type)) - - -def pop(data: bytes) -> Tuple[TSerializable, bytes]: - """ - This function parses a tnetstring into a python object. - It returns a tuple giving the parsed object and a string - containing any unparsed data from the end of the string. - """ - # Parse out data length, type and remaining string. - try: - length, data = data.split(b':', 1) - length = int(length) - except ValueError: - raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data)) - try: - data, data_type, remain = data[:length], data[length], data[length + 1:] - except IndexError: - # This fires if len(data) < dlen, meaning we don't need - # to further validate that data is the right length. - raise ValueError("not a tnetstring: invalid length prefix: {}".format(length)) - # Parse the data based on the type tag. - return parse(data_type, data), remain - - -__all__ = ["dump", "dumps", "load", "loads", "pop"] diff --git a/mitmproxy/contrib/py3/tnetstring_tests.py b/mitmproxy/contrib/py3/tnetstring_tests.py deleted file mode 100644 index 4ee184d5..00000000 --- a/mitmproxy/contrib/py3/tnetstring_tests.py +++ /dev/null @@ -1,133 +0,0 @@ -import unittest -import random -import math -import io -from . import tnetstring -import struct - -MAXINT = 2 ** (struct.Struct('i').size * 8 - 1) - 1 - -FORMAT_EXAMPLES = { - b'0:}': {}, - b'0:]': [], - b'51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00\x00,]}': - {'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, - b'5:12345#': 12345, - b'12:this is cool,': b'this is cool', - b'0:,': b'', - b'0:~': None, - b'4:true!': True, - b'5:false!': False, - b'10:\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00,': b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', - b'24:5:12345#5:67890#5:xxxxx,]': [12345, 67890, b'xxxxx'], - b'18:3:0.1^3:0.2^3:0.3^]': [0.1, 0.2, 0.3], - b'243:238:233:228:223:218:213:208:203:198:193:188:183:178:173:168:163:158:153:148:143:138:133:128:123:118:113:108:103:99:95:91:87:83:79:75:71:67:63:59:55:51:47:43:39:35:31:27:23:19:15:11:hello-there,]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]': [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[b'hello-there']]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] -} - -def get_random_object(random=random, depth=0): - """Generate a random serializable object.""" - # The probability of generating a scalar value increases as the depth increase. - # This ensures that we bottom out eventually. - if random.randint(depth,10) <= 4: - what = random.randint(0,1) - if what == 0: - n = random.randint(0,10) - l = [] - for _ in range(n): - l.append(get_random_object(random,depth+1)) - return l - if what == 1: - n = random.randint(0,10) - d = {} - for _ in range(n): - n = random.randint(0,100) - k = str([random.randint(32,126) for _ in range(n)]) - d[k] = get_random_object(random,depth+1) - return d - else: - what = random.randint(0,4) - if what == 0: - return None - if what == 1: - return True - if what == 2: - return False - if what == 3: - if random.randint(0,1) == 0: - return random.randint(0,MAXINT) - else: - return -1 * random.randint(0,MAXINT) - n = random.randint(0,100) - return bytes([random.randint(32,126) for _ in range(n)]) - -class Test_Format(unittest.TestCase): - def test_roundtrip_format_examples(self): - for data, expect in FORMAT_EXAMPLES.items(): - self.assertEqual(expect,tnetstring.loads(data)) - self.assertEqual(expect,tnetstring.loads(tnetstring.dumps(expect))) - self.assertEqual((expect,b''),tnetstring.pop(data)) - - def test_roundtrip_format_random(self): - for _ in range(500): - v = get_random_object() - self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v))) - self.assertEqual((v,b""),tnetstring.pop(tnetstring.dumps(v))) - - def test_unicode_handling(self): - with self.assertRaises(ValueError): - tnetstring.dumps("hello") - self.assertEqual(tnetstring.dumps("hello".encode()),b"5:hello,") - self.assertEqual(type(tnetstring.loads(b"5:hello,")),bytes) - - def test_roundtrip_format_unicode(self): - for _ in range(500): - v = get_random_object() - self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v))) - self.assertEqual((v,b''),tnetstring.pop(tnetstring.dumps(v))) - - def test_roundtrip_big_integer(self): - i1 = math.factorial(30000) - s = tnetstring.dumps(i1) - i2 = tnetstring.loads(s) - self.assertEqual(i1, i2) - -class Test_FileLoading(unittest.TestCase): - def test_roundtrip_file_examples(self): - for data, expect in FORMAT_EXAMPLES.items(): - s = io.BytesIO() - s.write(data) - s.write(b'OK') - s.seek(0) - self.assertEqual(expect,tnetstring.load(s)) - self.assertEqual(b'OK',s.read()) - s = io.BytesIO() - tnetstring.dump(expect,s) - s.write(b'OK') - s.seek(0) - self.assertEqual(expect,tnetstring.load(s)) - self.assertEqual(b'OK',s.read()) - - def test_roundtrip_file_random(self): - for _ in range(500): - v = get_random_object() - s = io.BytesIO() - tnetstring.dump(v,s) - s.write(b'OK') - s.seek(0) - self.assertEqual(v,tnetstring.load(s)) - self.assertEqual(b'OK',s.read()) - - def test_error_on_absurd_lengths(self): - s = io.BytesIO() - s.write(b'1000000000:pwned!,') - s.seek(0) - with self.assertRaises(ValueError): - tnetstring.load(s) - self.assertEqual(s.read(1),b':') - -def suite(): - loader = unittest.TestLoader() - suite = unittest.TestSuite() - suite.addTest(loader.loadTestsFromTestCase(Test_Format)) - suite.addTest(loader.loadTestsFromTestCase(Test_FileLoading)) - return suite \ No newline at end of file diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index 1ebaba21..5fc26b45 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -1,8 +1,254 @@ +""" +tnetstring: data serialization using typed netstrings +====================================================== + +This is a custom Python 3 implementation of tnetstrings. +Compared to other implementations, the main difference +is that this implementation supports a custom unicode datatype. + +An ordinary tnetstring is a blob of data prefixed with its length and postfixed +with its type. Here are some examples: + + >>> tnetstring.dumps("hello world") + 11:hello world, + >>> tnetstring.dumps(12345) + 5:12345# + >>> tnetstring.dumps([12345, True, 0]) + 19:5:12345#4:true!1:0#] + +This module gives you the following functions: + + :dump: dump an object as a tnetstring to a file + :dumps: dump an object as a tnetstring to a string + :load: load a tnetstring-encoded object from a file + :loads: load a tnetstring-encoded object from a string + +Note that since parsing a tnetstring requires reading all the data into memory +at once, there's no efficiency gain from using the file-based versions of these +functions. They're only here so you can use load() to read precisely one +item from a file or socket without consuming any extra data. + +The tnetstrings specification explicitly states that strings are binary blobs +and forbids the use of unicode at the protocol level. +**This implementation decodes dictionary keys as surrogate-escaped ASCII**, +all other strings are returned as plain bytes. + +:Copyright: (c) 2012-2013 by Ryan Kelly . +:Copyright: (c) 2014 by Carlo Pires . +:Copyright: (c) 2016 by Maximilian Hils . + +:License: MIT +""" + +import collections import six +from typing import io, Union, Tuple # noqa + +TSerializable = Union[None, bool, int, float, bytes, list, tuple, dict] + + +def dumps(value): + # type: (TSerializable) -> bytes + """ + This function dumps a python object as a tnetstring. + """ + # This uses a deque to collect output fragments in reverse order, + # then joins them together at the end. It's measurably faster + # than creating all the intermediate strings. + q = collections.deque() + _rdumpq(q, 0, value) + return b''.join(q) + + +def dump(value, file_handle): + # type: (TSerializable, io.BinaryIO) -> None + """ + This function dumps a python object as a tnetstring and + writes it to the given file. + """ + file_handle.write(dumps(value)) + + +def _rdumpq(q, size, value): + # type: (collections.deque, int, TSerializable) -> int + """ + Dump value as a tnetstring, to a deque instance, last chunks first. + + This function generates the tnetstring representation of the given value, + pushing chunks of the output onto the given deque instance. It pushes + the last chunk first, then recursively generates more chunks. + + When passed in the current size of the string in the queue, it will return + the new size of the string in the queue. + + Operating last-chunk-first makes it easy to calculate the size written + for recursive structures without having to build their representation as + a string. This is measurably faster than generating the intermediate + strings, especially on deeply nested structures. + """ + write = q.appendleft + if value is None: + write(b'0:~') + return size + 3 + elif value is True: + write(b'4:true!') + return size + 7 + elif value is False: + write(b'5:false!') + return size + 8 + elif isinstance(value, int): + data = str(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s#' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, float): + # Use repr() for float rather than str(). + # It round-trips more accurately. + # Probably unnecessary in later python versions that + # use David Gay's ftoa routines. + data = repr(value).encode() + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s^' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, bytes): + data = value + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s,' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, six.text_type): + data = value.encode() + ldata = len(data) + span = str(ldata).encode() + write(b'%s:%s;' % (span, data)) + return size + 2 + len(span) + ldata + elif isinstance(value, (list, tuple)): + write(b']') + init_size = size = size + 1 + for item in reversed(value): + size = _rdumpq(q, size, item) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + elif isinstance(value, dict): + write(b'}') + init_size = size = size + 1 + for (k, v) in value.items(): + if isinstance(k, str): + k = k.encode("ascii", "strict") + size = _rdumpq(q, size, v) + size = _rdumpq(q, size, k) + span = str(size - init_size).encode() + write(b':') + write(span) + return size + 1 + len(span) + else: + raise ValueError("unserializable object: {} ({})".format(value, type(value))) + + +def loads(string): + # type: (bytes) -> TSerializable + """ + This function parses a tnetstring into a python object. + """ + return pop(string)[0] + + +def load(file_handle): + # type: (io.BinaryIO) -> TSerializable + """load(file) -> object + + This function reads a tnetstring from a file and parses it into a + python object. The file must support the read() method, and this + function promises not to read more data than necessary. + """ + # Read the length prefix one char at a time. + # Note that the netstring spec explicitly forbids padding zeros. + c = file_handle.read(1) + data_length = b"" + while c.isdigit(): + data_length += c + if len(data_length) > 9: + raise ValueError("not a tnetstring: absurdly large length prefix") + c = file_handle.read(1) + if c != b":": + raise ValueError("not a tnetstring: missing or invalid length prefix") + + data = file_handle.read(int(data_length)) + data_type = file_handle.read(1)[0] + + return parse(data_type, data) + + +def parse(data_type, data): + # type: (int, bytes) -> TSerializable + if data_type == ord(b','): + return data + if data_type == ord(b';'): + return data.decode() + if data_type == ord(b'#'): + try: + return int(data) + except ValueError: + raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) + if data_type == ord(b'^'): + try: + return float(data) + except ValueError: + raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) + if data_type == ord(b'!'): + if data == b'true': + return True + elif data == b'false': + return False + else: + raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) + if data_type == ord(b'~'): + if data: + raise ValueError("not a tnetstring: invalid null literal") + return None + if data_type == ord(b']'): + l = [] + while data: + item, data = pop(data) + l.append(item) + return l + if data_type == ord(b'}'): + d = {} + while data: + key, data = pop(data) + if isinstance(key, bytes): + key = key.decode("ascii", "strict") + val, data = pop(data) + d[key] = val + return d + raise ValueError("unknown type tag: {}".format(data_type)) + + +def pop(data): + # type: (bytes) -> Tuple[TSerializable, bytes] + """ + This function parses a tnetstring into a python object. + It returns a tuple giving the parsed object and a string + containing any unparsed data from the end of the string. + """ + # Parse out data length, type and remaining string. + try: + length, data = data.split(b':', 1) + length = int(length) + except ValueError: + raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data)) + try: + data, data_type, remain = data[:length], data[length], data[length + 1:] + except IndexError: + # This fires if len(data) < dlen, meaning we don't need + # to further validate that data is the right length. + raise ValueError("not a tnetstring: invalid length prefix: {}".format(length)) + # Parse the data based on the type tag. + return parse(data_type, data), remain -if six.PY2: - from .py2.tnetstring import load, loads, dump, dumps, pop -else: - from .py3.tnetstring import load, loads, dump, dumps, pop -__all__ = ["load", "loads", "dump", "dumps", "pop"] +__all__ = ["dump", "dumps", "load", "loads", "pop"] -- cgit v1.2.3 From d406bee988dc01126cfbdfc938b561e10b518610 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 5 Jul 2016 20:28:13 -0700 Subject: tnetstring3: adapt to unicode support --- mitmproxy/contrib/tnetstring.py | 14 +++--- mitmproxy/flow/io.py | 7 ++- mitmproxy/flow/io_compat.py | 76 +++++++++++++++++++++---------- mitmproxy/models/connections.py | 2 +- test/mitmproxy/test_contrib_tnetstring.py | 8 +--- 5 files changed, 66 insertions(+), 41 deletions(-) diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index 5fc26b45..0383f98e 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -96,7 +96,7 @@ def _rdumpq(q, size, value): elif value is False: write(b'5:false!') return size + 8 - elif isinstance(value, int): + elif isinstance(value, six.integer_types): data = str(value).encode() ldata = len(data) span = str(ldata).encode() @@ -119,7 +119,7 @@ def _rdumpq(q, size, value): write(b'%s:%s,' % (span, data)) return size + 2 + len(span) + ldata elif isinstance(value, six.text_type): - data = value.encode() + data = value.encode("utf8") ldata = len(data) span = str(ldata).encode() write(b'%s:%s;' % (span, data)) @@ -137,8 +137,6 @@ def _rdumpq(q, size, value): write(b'}') init_size = size = size + 1 for (k, v) in value.items(): - if isinstance(k, str): - k = k.encode("ascii", "strict") size = _rdumpq(q, size, v) size = _rdumpq(q, size, k) span = str(size - init_size).encode() @@ -184,13 +182,17 @@ def load(file_handle): def parse(data_type, data): + if six.PY2: + data_type = ord(data_type) # type: (int, bytes) -> TSerializable if data_type == ord(b','): return data if data_type == ord(b';'): - return data.decode() + return data.decode("utf8") if data_type == ord(b'#'): try: + if six.PY2: + return long(data) return int(data) except ValueError: raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) @@ -220,8 +222,6 @@ def parse(data_type, data): d = {} while data: key, data = pop(data) - if isinstance(key, bytes): - key = key.decode("ascii", "strict") val, data = pop(data) d[key] = val return d diff --git a/mitmproxy/flow/io.py b/mitmproxy/flow/io.py index e5716940..276d7a5b 100644 --- a/mitmproxy/flow/io.py +++ b/mitmproxy/flow/io.py @@ -44,10 +44,9 @@ class FlowReader: raise exceptions.FlowReadException(str(e)) if can_tell: off = self.fo.tell() - data_type = data["type"].decode() - if data_type not in models.FLOW_TYPES: - raise exceptions.FlowReadException("Unknown flow type: {}".format(data_type)) - yield models.FLOW_TYPES[data_type].from_state(data) + if data["type"] not in models.FLOW_TYPES: + raise exceptions.FlowReadException("Unknown flow type: {}".format(data["type"])) + yield models.FLOW_TYPES[data["type"]].from_state(data) except ValueError: # Error is due to EOF if can_tell and self.fo.tell() == off and self.fo.read() == b'': diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index 55971f5e..1e67dde4 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -3,44 +3,74 @@ This module handles the import of mitmproxy flows generated by old versions. """ from __future__ import absolute_import, print_function, division +import six + from netlib import version def convert_013_014(data): - data["request"]["first_line_format"] = data["request"].pop("form_in") - data["request"]["http_version"] = "HTTP/" + ".".join(str(x) for x in data["request"].pop("httpversion")) - data["response"]["http_version"] = "HTTP/" + ".".join(str(x) for x in data["response"].pop("httpversion")) - data["response"]["status_code"] = data["response"].pop("code") - data["response"]["body"] = data["response"].pop("content") - data["server_conn"].pop("state") - data["server_conn"]["via"] = None - data["version"] = (0, 14) + data[b"request"][b"first_line_format"] = data[b"request"].pop(b"form_in") + data[b"request"][b"http_version"] = b"HTTP/" + ".".join(str(x) for x in data[b"request"].pop(b"httpversion")).encode() + data[b"response"][b"http_version"] = b"HTTP/" + ".".join(str(x) for x in data[b"response"].pop(b"httpversion")).encode() + data[b"response"][b"status_code"] = data[b"response"].pop(b"code") + data[b"response"][b"body"] = data[b"response"].pop(b"content") + data[b"server_conn"].pop(b"state") + data[b"server_conn"][b"via"] = None + data[b"version"] = (0, 14) return data def convert_014_015(data): - data["version"] = (0, 15) + data[b"version"] = (0, 15) return data def convert_015_016(data): - for m in ("request", "response"): - if "body" in data[m]: - data[m]["content"] = data[m].pop("body") - if "msg" in data["response"]: - data["response"]["reason"] = data["response"].pop("msg") - data["request"].pop("form_out", None) - data["version"] = (0, 16) + for m in (b"request", b"response"): + if b"body" in data[m]: + data[m][b"content"] = data[m].pop(b"body") + if b"msg" in data[b"response"]: + data[b"response"][b"reason"] = data[b"response"].pop(b"msg") + data[b"request"].pop(b"form_out", None) + data[b"version"] = (0, 16) return data def convert_016_017(data): - data["server_conn"]["peer_address"] = None - data["version"] = (0, 17) + data[b"server_conn"][b"peer_address"] = None + data[b"version"] = (0, 17) return data def convert_017_018(data): + if not six.PY2: + # Python 2 versions of mitmproxy did not support serializing unicode. + def dict_keys_to_str(o): + if isinstance(o, dict): + return {k.decode(): dict_keys_to_str(v) for k, v in o.items()} + else: + return o + data = dict_keys_to_str(data) + + def dict_vals_to_str(o, decode): + for k, v in decode.items(): + if not o or k not in o: + continue + if v is True: + o[k] = o[k].decode() + else: + dict_vals_to_str(o[k], v) + dict_vals_to_str(data, { + "type": True, + "id": True, + "request": { + "first_line_format": True + }, + "error": { + "msg": True + } + }) + data["server_conn"]["ip_address"] = data["server_conn"].pop("peer_address") data["version"] = (0, 18) return data @@ -57,13 +87,13 @@ converters = { def migrate_flow(flow_data): while True: - flow_version = tuple(flow_data["version"][:2]) - if flow_version == version.IVERSION[:2]: + flow_version = tuple(flow_data.get(b"version", flow_data.get("version"))) + if flow_version[:2] == version.IVERSION[:2]: break - elif flow_version in converters: - flow_data = converters[flow_version](flow_data) + elif flow_version[:2] in converters: + flow_data = converters[flow_version[:2]](flow_data) else: - v = ".".join(str(i) for i in flow_data["version"]) + v = ".".join(str(i) for i in flow_version) raise ValueError( "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v) ) diff --git a/mitmproxy/models/connections.py b/mitmproxy/models/connections.py index d71379bc..3e1a0928 100644 --- a/mitmproxy/models/connections.py +++ b/mitmproxy/models/connections.py @@ -162,7 +162,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): source_address=tcp.Address, ssl_established=bool, cert=certutils.SSLCert, - sni=str, + sni=bytes, timestamp_start=float, timestamp_tcp_setup=float, timestamp_ssl_setup=float, diff --git a/test/mitmproxy/test_contrib_tnetstring.py b/test/mitmproxy/test_contrib_tnetstring.py index 8ae35a25..908cec27 100644 --- a/test/mitmproxy/test_contrib_tnetstring.py +++ b/test/mitmproxy/test_contrib_tnetstring.py @@ -15,7 +15,9 @@ FORMAT_EXAMPLES = { {'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, b'5:12345#': 12345, b'12:this is cool,': b'this is cool', + b'19:this is unicode \xe2\x98\x85;': u'this is unicode \u2605', b'0:,': b'', + b'0:;': u'', b'0:~': None, b'4:true!': True, b'5:false!': False, @@ -78,12 +80,6 @@ class Test_Format(unittest.TestCase): self.assertEqual(v, tnetstring.loads(tnetstring.dumps(v))) self.assertEqual((v, b""), tnetstring.pop(tnetstring.dumps(v))) - def test_unicode_handling(self): - with self.assertRaises(ValueError): - tnetstring.dumps(u"hello") - self.assertEqual(tnetstring.dumps(u"hello".encode()), b"5:hello,") - self.assertEqual(type(tnetstring.loads(b"5:hello,")), bytes) - def test_roundtrip_format_unicode(self): for _ in range(500): v = get_random_object() -- cgit v1.2.3 From 2c37ebfc7215649cc633047c0b036de66d847af1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 13:24:50 -0700 Subject: fix dump file cross compat between python versions --- mitmproxy/contrib/tnetstring.py | 10 ++++- mitmproxy/flow/io_compat.py | 83 ++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index 0383f98e..d99a83f9 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -116,13 +116,19 @@ def _rdumpq(q, size, value): data = value ldata = len(data) span = str(ldata).encode() - write(b'%s:%s,' % (span, data)) + write(b',') + write(data) + write(b':') + write(span) return size + 2 + len(span) + ldata elif isinstance(value, six.text_type): data = value.encode("utf8") ldata = len(data) span = str(ldata).encode() - write(b'%s:%s;' % (span, data)) + write(b';') + write(data) + write(b':') + write(span) return size + 2 + len(span) + ldata elif isinstance(value, (list, tuple)): write(b']') diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index 1e67dde4..ec825f71 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -5,13 +5,15 @@ from __future__ import absolute_import, print_function, division import six -from netlib import version +from netlib import version, strutils def convert_013_014(data): data[b"request"][b"first_line_format"] = data[b"request"].pop(b"form_in") - data[b"request"][b"http_version"] = b"HTTP/" + ".".join(str(x) for x in data[b"request"].pop(b"httpversion")).encode() - data[b"response"][b"http_version"] = b"HTTP/" + ".".join(str(x) for x in data[b"response"].pop(b"httpversion")).encode() + data[b"request"][b"http_version"] = b"HTTP/" + ".".join( + str(x) for x in data[b"request"].pop(b"httpversion")).encode() + data[b"response"][b"http_version"] = b"HTTP/" + ".".join( + str(x) for x in data[b"response"].pop(b"httpversion")).encode() data[b"response"][b"status_code"] = data[b"response"].pop(b"code") data[b"response"][b"body"] = data[b"response"].pop(b"content") data[b"server_conn"].pop(b"state") @@ -43,39 +45,57 @@ def convert_016_017(data): def convert_017_018(data): - if not six.PY2: - # Python 2 versions of mitmproxy did not support serializing unicode. - def dict_keys_to_str(o): - if isinstance(o, dict): - return {k.decode(): dict_keys_to_str(v) for k, v in o.items()} - else: - return o - data = dict_keys_to_str(data) - - def dict_vals_to_str(o, decode): - for k, v in decode.items(): - if not o or k not in o: - continue - if v is True: - o[k] = o[k].decode() - else: - dict_vals_to_str(o[k], v) - dict_vals_to_str(data, { - "type": True, - "id": True, - "request": { - "first_line_format": True - }, - "error": { - "msg": True - } - }) + # convert_unicode needs to be called for every dual release and the first py3-only release + data = convert_unicode(data) data["server_conn"]["ip_address"] = data["server_conn"].pop("peer_address") data["version"] = (0, 18) return data +def _convert_dict_keys(o): + # type: (Any) -> Any + if isinstance(o, dict): + return {strutils.native(k): _convert_dict_keys(v) for k, v in o.items()} + else: + return o + + +def _convert_dict_vals(o, values_to_convert): + # type: (dict, dict) -> dict + for k, v in values_to_convert.items(): + if not o or k not in o: + continue + if v is True: + o[k] = strutils.native(o[k]) + else: + _convert_dict_vals(o[k], v) + return o + + +def convert_unicode(data): + # type: (dict) -> dict + """ + The Python 2 version of mitmproxy serializes everything as bytes. + This method converts between Python 3 and Python 2 dumpfiles. + """ + if not six.PY2: + data = _convert_dict_keys(data) + data = _convert_dict_vals( + data, { + "type": True, + "id": True, + "request": { + "first_line_format": True + }, + "error": { + "msg": True + } + } + ) + return data + + converters = { (0, 13): convert_013_014, (0, 14): convert_014_015, @@ -97,4 +117,7 @@ def migrate_flow(flow_data): raise ValueError( "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v) ) + # TODO: This should finally be moved in the converter for the first py3-only release. + # It's here so that a py2 0.18 dump can be read by py3 0.18 and vice versa. + flow_data = convert_unicode(flow_data) return flow_data -- cgit v1.2.3 From 8287ce7e6dcf31e65519629bb064044a44de46d1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 16:36:04 -0700 Subject: fix tests --- test/mitmproxy/test_contrib_tnetstring.py | 2 +- test/mitmproxy/tutils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mitmproxy/test_contrib_tnetstring.py b/test/mitmproxy/test_contrib_tnetstring.py index 908cec27..05c4a7c9 100644 --- a/test/mitmproxy/test_contrib_tnetstring.py +++ b/test/mitmproxy/test_contrib_tnetstring.py @@ -12,7 +12,7 @@ FORMAT_EXAMPLES = { b'0:}': {}, b'0:]': [], b'51:5:hello,39:11:12345678901#4:this,4:true!0:~4:\x00\x00\x00\x00,]}': - {'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, + {b'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, b'5:12345#': 12345, b'12:this is cool,': b'this is cool', b'19:this is unicode \xe2\x98\x85;': u'this is unicode \u2605', diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index d0a09035..5aade60c 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -130,7 +130,7 @@ def tserver_conn(): timestamp_ssl_setup=3, timestamp_end=4, ssl_established=False, - sni="address", + sni=b"address", via=None )) c.reply = controller.DummyReply() -- cgit v1.2.3 From 64a867973d5bac136c2e1c3c11c457d6b04d6649 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 21:03:17 -0700 Subject: sni is now str, not bytes --- mitmproxy/models/connections.py | 7 ++++--- mitmproxy/models/flow.py | 16 ++++++---------- mitmproxy/protocol/tls.py | 13 ++++++++----- netlib/tcp.py | 4 ++-- netlib/utils.py | 4 +--- pathod/pathod.py | 5 ++++- test/mitmproxy/test_server.py | 6 +++--- test/mitmproxy/tutils.py | 2 +- test/netlib/test_tcp.py | 26 +++++++++++++------------- test/pathod/test_pathoc.py | 4 ++-- 10 files changed, 44 insertions(+), 43 deletions(-) diff --git a/mitmproxy/models/connections.py b/mitmproxy/models/connections.py index 3e1a0928..570e89a9 100644 --- a/mitmproxy/models/connections.py +++ b/mitmproxy/models/connections.py @@ -8,7 +8,6 @@ import six from mitmproxy import stateobject from netlib import certutils -from netlib import strutils from netlib import tcp @@ -162,7 +161,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): source_address=tcp.Address, ssl_established=bool, cert=certutils.SSLCert, - sni=bytes, + sni=str, timestamp_start=float, timestamp_tcp_setup=float, timestamp_ssl_setup=float, @@ -206,6 +205,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.wfile.flush() def establish_ssl(self, clientcerts, sni, **kwargs): + if sni and not isinstance(sni, six.string_types): + raise ValueError("sni must be str, not " + type(sni).__name__) clientcert = None if clientcerts: if os.path.isfile(clientcerts): @@ -217,7 +218,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - self.convert_to_ssl(cert=clientcert, sni=strutils.always_bytes(sni), **kwargs) + self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = time.time() diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py index 0e4f80cb..f4993b7a 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/models/flow.py @@ -9,6 +9,7 @@ from mitmproxy.models.connections import ClientConnection from mitmproxy.models.connections import ServerConnection from netlib import version +from typing import Optional # noqa class Error(stateobject.StateObject): @@ -70,18 +71,13 @@ class Flow(stateobject.StateObject): def __init__(self, type, client_conn, server_conn, live=None): self.type = type self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" + self.client_conn = client_conn # type: ClientConnection + self.server_conn = server_conn # type: ServerConnection self.live = live - """@type: LiveConnection""" - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None + self.error = None # type: Error + self.intercepted = False # type: bool + self._backup = None # type: Optional[Flow] self.reply = None _stateobject_attributes = dict( diff --git a/mitmproxy/protocol/tls.py b/mitmproxy/protocol/tls.py index 9f883b2b..8ef34493 100644 --- a/mitmproxy/protocol/tls.py +++ b/mitmproxy/protocol/tls.py @@ -10,6 +10,7 @@ import netlib.exceptions from mitmproxy import exceptions from mitmproxy.contrib.tls import _constructs from mitmproxy.protocol import base +from netlib import utils # taken from https://testssl.sh/openssl-rfc.mappping.html @@ -274,10 +275,11 @@ class TlsClientHello(object): is_valid_sni_extension = ( extension.type == 0x00 and len(extension.server_names) == 1 and - extension.server_names[0].type == 0 + extension.server_names[0].type == 0 and + utils.is_valid_host(extension.server_names[0].name) ) if is_valid_sni_extension: - return extension.server_names[0].name + return extension.server_names[0].name.decode("idna") @property def alpn_protocols(self): @@ -403,13 +405,14 @@ class TlsLayer(base.Layer): self._establish_tls_with_server() def set_server_tls(self, server_tls, sni=None): + # type: (bool, Union[six.text_type, None, False]) -> None """ Set the TLS settings for the next server connection that will be established. This function will not alter an existing connection. Args: server_tls: Shall we establish TLS with the server? - sni: ``bytes`` for a custom SNI value, + sni: ``str`` for a custom SNI value, ``None`` for the client SNI value, ``False`` if no SNI value should be sent. """ @@ -602,9 +605,9 @@ class TlsLayer(base.Layer): host = upstream_cert.cn.decode("utf8").encode("idna") # Also add SNI values. if self._client_hello.sni: - sans.add(self._client_hello.sni) + sans.add(self._client_hello.sni.encode("idna")) if self._custom_server_sni: - sans.add(self._custom_server_sni) + sans.add(self._custom_server_sni.encode("idna")) # RFC 2818: If a subjectAltName extension of type dNSName is present, that MUST be used as the identity. # In other words, the Common Name is irrelevant then. diff --git a/netlib/tcp.py b/netlib/tcp.py index 69dafc1f..cf099edd 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -676,7 +676,7 @@ class TCPClient(_Connection): self.connection = SSL.Connection(context, self.connection) if sni: self.sni = sni - self.connection.set_tlsext_host_name(sni) + self.connection.set_tlsext_host_name(sni.encode("idna")) self.connection.set_connect_state() try: self.connection.do_handshake() @@ -705,7 +705,7 @@ class TCPClient(_Connection): if self.cert.cn: crt["subject"] = [[["commonName", self.cert.cn.decode("ascii", "strict")]]] if sni: - hostname = sni.decode("ascii", "strict") + hostname = sni else: hostname = "no-hostname" ssl_match_hostname.match_hostname(crt, hostname) diff --git a/netlib/utils.py b/netlib/utils.py index 79340cbd..23c16dc3 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -73,11 +73,9 @@ _label_valid = re.compile(b"(?!-)[A-Z\d-]{1,63}(? bool """ Checks if a hostname is valid. - - Args: - host (bytes): The hostname """ try: host.decode("idna") diff --git a/pathod/pathod.py b/pathod/pathod.py index 3df86aae..7087cba6 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -89,7 +89,10 @@ class PathodHandler(tcp.BaseHandler): self.http2_framedump = http2_framedump def handle_sni(self, connection): - self.sni = connection.get_servername() + sni = connection.get_servername() + if sni: + sni = sni.decode("idna") + self.sni = sni def http_serve_crafted(self, crafted, logctx): error, crafted = self.server.check_policy( diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 1bbef975..0ab7624e 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -100,10 +100,10 @@ class CommonMixin: if not self.ssl: return - f = self.pathod("304", sni=b"testserver.com") + f = self.pathod("304", sni="testserver.com") assert f.status_code == 304 log = self.server.last_log() - assert log["request"]["sni"] == b"testserver.com" + assert log["request"]["sni"] == "testserver.com" class TcpMixin: @@ -498,7 +498,7 @@ class TestHttps2Http(tservers.ReverseProxyTest): assert p.request("get:'/p/200'").status_code == 200 def test_sni(self): - p = self.pathoc(ssl=True, sni=b"example.com") + p = self.pathoc(ssl=True, sni="example.com") assert p.request("get:'/p/200'").status_code == 200 assert all("Error in handle_sni" not in msg for msg in self.proxy.tlog) diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index 5aade60c..d0a09035 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -130,7 +130,7 @@ def tserver_conn(): timestamp_ssl_setup=3, timestamp_end=4, ssl_established=False, - sni=b"address", + sni="address", via=None )) c.reply = controller.DummyReply() diff --git a/test/netlib/test_tcp.py b/test/netlib/test_tcp.py index 590bcc01..273427d5 100644 --- a/test/netlib/test_tcp.py +++ b/test/netlib/test_tcp.py @@ -169,7 +169,7 @@ class TestServerSSL(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com", options=SSL.OP_ALL) + c.convert_to_ssl(sni="foo.com", options=SSL.OP_ALL) testval = b"echo!\n" c.wfile.write(testval) c.wfile.flush() @@ -179,7 +179,7 @@ class TestServerSSL(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): assert not c.get_current_cipher() - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") ret = c.get_current_cipher() assert ret assert "AES" in ret[0] @@ -195,7 +195,7 @@ class TestSSLv3Only(tservers.ServerTestBase): def test_failure(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - tutils.raises(TlsException, c.convert_to_ssl, sni=b"foo.com") + tutils.raises(TlsException, c.convert_to_ssl, sni="foo.com") class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): @@ -238,7 +238,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): with c.connect(): with tutils.raises(InvalidCertificateException): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -272,7 +272,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): with c.connect(): with tutils.raises(InvalidCertificateException): c.convert_to_ssl( - sni=b"mitmproxy.org", + sni="mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -291,7 +291,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -307,7 +307,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_path=tutils.test_data.path("data/verificationcerts/") ) @@ -371,8 +371,8 @@ class TestSNI(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") - assert c.sni == b"foo.com" + c.convert_to_ssl(sni="foo.com") + assert c.sni == "foo.com" assert c.rfile.readline() == b"foo.com" @@ -385,7 +385,7 @@ class TestServerCipherList(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") assert c.rfile.readline() == b"['RC4-SHA']" @@ -405,7 +405,7 @@ class TestServerCurrentCipher(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") assert b"RC4-SHA" in c.rfile.readline() @@ -418,7 +418,7 @@ class TestServerCipherListError(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - tutils.raises("handshake error", c.convert_to_ssl, sni=b"foo.com") + tutils.raises("handshake error", c.convert_to_ssl, sni="foo.com") class TestClientCipherListError(tservers.ServerTestBase): @@ -433,7 +433,7 @@ class TestClientCipherListError(tservers.ServerTestBase): tutils.raises( "cipher specification", c.convert_to_ssl, - sni=b"foo.com", + sni="foo.com", cipher_list="bogus" ) diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 28f9f0f8..361a863b 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -54,10 +54,10 @@ class TestDaemonSSL(PathocTestDaemon): def test_sni(self): self.tval( ["get:/p/200"], - sni=b"foobar.com" + sni="foobar.com" ) log = self.d.log() - assert log[0]["request"]["sni"] == b"foobar.com" + assert log[0]["request"]["sni"] == "foobar.com" def test_showssl(self): assert "certificate chain" in self.tval(["get:/p/200"], showssl=True) -- cgit v1.2.3 From 275cf7a6c2189859ad5f5719c8b74b1bc6eebbf2 Mon Sep 17 00:00:00 2001 From: Linmiao Xu Date: Wed, 6 Jul 2016 21:01:04 -0700 Subject: Detect JSON API media type as a json content view IANA assignment here: https://www.iana.org/assignments/media-types/application/vnd.api+json More about JSON API: https://github.com/json-api/json-api --- mitmproxy/contentviews.py | 5 ++++- test/mitmproxy/test_contentview.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 6f64e360..331a706f 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -226,7 +226,10 @@ class ViewXML(View): class ViewJSON(View): name = "JSON" prompt = ("json", "s") - content_types = ["application/json"] + content_types = [ + "application/json", + "application/vnd.api+json" + ] def __call__(self, data, **metadata): pj = pretty_json(data) diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index 52fceeac..c11a5fe5 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -201,6 +201,13 @@ Larry ) assert "Raw" in r[0] + r = cv.get_content_view( + cv.get("Auto"), + b"[1, 2, 3]", + headers=Headers(content_type="application/vnd.api+json") + ) + assert r[0] == "JSON" + tutils.raises( ContentViewException, cv.get_content_view, -- cgit v1.2.3 From f259b9478fb03e816af8abde29ae369342d40e4f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 01:21:15 -0700 Subject: disable harparser on py3 --- examples/har_extractor.py | 6 ++++++ test/mitmproxy/test_examples.py | 7 +++++++ tox.ini | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/har_extractor.py b/examples/har_extractor.py index d6b50c21..a5c05519 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -60,6 +60,12 @@ def start(context): suit your actual needs of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them from time to time. """ + if sys.version_info >= (3, 0): + raise RuntimeError( + "har_extractor.py does not work on Python 3. " + "Please check out https://github.com/mitmproxy/mitmproxy/issues/1320 " + "if you want to help making this work again." + ) context.dump_file = None if len(sys.argv) > 1: context.dump_file = sys.argv[1] diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 22d3c425..206a0366 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,6 +1,7 @@ import glob import json import os +import sys from contextlib import contextmanager from mitmproxy import script @@ -133,6 +134,12 @@ def test_redirect_requests(): def test_har_extractor(): + if sys.version_info >= (3, 0): + with tutils.raises("does not work on Python 3"): + with example("har_extractor.py -"): + pass + return + with tutils.raises(script.ScriptException): with example("har_extractor.py"): pass diff --git a/tox.ini b/tox.ini index 251609a5..6e9aa386 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From a212ce1f636bf14fdfb7cd7f238626c70bcfacb9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 02:30:25 -0700 Subject: py3++ --- mitmproxy/console/common.py | 3 ++- mitmproxy/console/flowdetailview.py | 2 +- mitmproxy/console/flowview.py | 2 +- mitmproxy/models/http.py | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index b450c19d..470db88d 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -4,6 +4,7 @@ import os import urwid import urwid.util +import six import netlib from mitmproxy import flow @@ -108,7 +109,7 @@ def shortcuts(k): def fcol(s, attr): - s = unicode(s) + s = six.text_type(s) return ( "fixed", len(s), diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/console/flowdetailview.py index 2a493b90..0a03e1c4 100644 --- a/mitmproxy/console/flowdetailview.py +++ b/mitmproxy/console/flowdetailview.py @@ -71,7 +71,7 @@ def flowdetails(state, flow): parts.append( [ "Alt names", - ", ".join(c.altnames) + ", ".join(str(x) for x in c.altnames) ] ) text.extend( diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index e9b23176..d809cf34 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -257,7 +257,7 @@ class FlowView(tabs.Tabs): def conn_text(self, conn): if conn: txt = common.format_keyvals( - [(h + ":", v) for (h, v) in conn.headers.fields], + [(h + ":", v) for (h, v) in conn.headers.items(multi=True)], key = "header", val = "text" ) diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 01f5f1ee..8aeb84d6 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -220,7 +220,7 @@ class HTTPFlow(Flow): If f is a string, it will be compiled as a filter expression. If the expression is invalid, ValueError is raised. """ - if isinstance(f, basestring): + if isinstance(f, str): from .. import filt f = filt.parse(f) -- cgit v1.2.3 From c870d26bc7fbb1b338a9e4bcb1e60d090bf4d99b Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 2 Jul 2016 18:18:50 +0530 Subject: py3++: test_protocol_http2 --- test/mitmproxy/test_protocol_http2.py | 6 +++--- tox.ini | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 2eb0b120..46d3dc05 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -126,7 +126,7 @@ class _Http2TestBase(object): client.wfile.flush() # read CONNECT response - while client.rfile.readline() != "\r\n": + while client.rfile.readline() != b"\r\n": pass client.convert_to_ssl(alpn_protos=[b'h2']) @@ -203,7 +203,7 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): (':path', '/'), ('ClIeNt-FoO', 'client-bar-1'), ('ClIeNt-FoO', 'client-bar-2'), - ], body='my request body echoed back to me') + ], body=b'my request body echoed back to me') done = False while not done: @@ -275,7 +275,7 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): (':scheme', 'https'), (':path', '/'), ], - body='foobar with request body', + body=b'foobar with request body', ) done = False diff --git a/tox.ini b/tox.ini index 6e9aa386..4dbc67e4 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py test/mitmproxy/test_protocol_http2.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From d962bd4e836856522c101b20e0167c3492d18965 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 7 Jul 2016 16:39:20 +0200 Subject: py3++ --- mitmproxy/protocol/http2.py | 6 +++--- test/mitmproxy/test_protocol_http2.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index b6623aa3..83ff71e4 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -197,7 +197,7 @@ class Http2Layer(base.Layer): self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) self.client_conn.send(self.client_conn.h2.data_to_send()) - headers = netlib.http.Headers([[str(k), str(v)] for k, v in event.headers]) + headers = netlib.http.Headers([[k, v] for k, v in event.headers]) self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, event.pushed_stream_id, headers) self.streams[event.pushed_stream_id].timestamp_start = time.time() self.streams[event.pushed_stream_id].pushed = True @@ -434,7 +434,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.server_conn.h2.safe_send_body( self.is_zombie, self.server_stream_id, - message.body + [message.body] ) if self.zombie: # pragma: no cover @@ -453,7 +453,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) return models.HTTPResponse( http_version=b"HTTP/2.0", status_code=status_code, - reason='', + reason=b'', headers=headers, content=None, timestamp_start=self.timestamp_start, diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 46d3dc05..58ffb787 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -169,8 +169,8 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): if isinstance(event, h2.events.ConnectionTerminated): return False elif isinstance(event, h2.events.RequestReceived): - assert ('client-foo', 'client-bar-1') in event.headers - assert ('client-foo', 'client-bar-2') in event.headers + assert (b'client-foo', b'client-bar-1') in event.headers + assert (b'client-foo', b'client-bar-2') in event.headers import warnings with warnings.catch_warnings(): @@ -538,7 +538,7 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): (':status', '200'), ('X-Stream-ID', str(event.stream_id)), ]) - h2_conn.send_data(event.stream_id, b'Stream-ID {}'.format(event.stream_id)) + h2_conn.send_data(event.stream_id, 'Stream-ID {}'.format(event.stream_id).encode()) h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() @@ -579,7 +579,7 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): assert len(self.master.state.flows) == len(new_streams) for flow in self.master.state.flows: assert flow.response.status_code == 200 - assert "Stream-ID" in flow.response.body + assert b"Stream-ID " in flow.response.body @requires_alpn @@ -598,7 +598,7 @@ class TestConnectionTerminated(_Http2TestBase, _Http2ServerBase): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.RequestReceived): - h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data='foobar') + h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data=b'foobar') wfile.write(h2_conn.data_to_send()) wfile.flush() return True @@ -630,4 +630,4 @@ class TestConnectionTerminated(_Http2TestBase, _Http2ServerBase): assert connection_terminated_event is not None assert connection_terminated_event.error_code == 5 assert connection_terminated_event.last_stream_id == 42 - assert connection_terminated_event.additional_data == 'foobar' + assert connection_terminated_event.additional_data == b'foobar' -- cgit v1.2.3 From acca8e5253878d6a346a15a0f64154bdbde49a98 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 7 Jul 2016 20:56:39 +0200 Subject: reorder travis jobs --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 345d6a58..a0e3c370 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ addons: packages: - libssl-dev -env: +env: global: - CI_DEPS=codecov>=2.0.5 - CI_COMMANDS=codecov @@ -18,22 +18,22 @@ env: matrix: fast_finish: true include: - - python: 2.7 - env: TOXENV=py27 - - python: 2.7 - env: TOXENV=py27 NO_ALPN=1 + - python: 3.5 + env: TOXENV=lint - python: 3.5 env: TOXENV=py35 - python: 3.5 env: TOXENV=py35 NO_ALPN=1 + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=py27 NO_ALPN=1 - language: generic os: osx osx_image: xcode7.1 git: depth: 9999999 env: TOXENV=py27 - - python: 3.5 - env: TOXENV=lint - python: 3.5 env: TOXENV=docs allow_failures: -- cgit v1.2.3 From f62e976e1e0245665aeeb08fa801661d6c766ba8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 17:29:22 -0700 Subject: py3++ --- mitmproxy/flow/modules.py | 5 +- netlib/http/message.py | 3 + netlib/http/request.py | 14 +++- netlib/http/response.py | 7 ++ test/mitmproxy/test_flow.py | 163 ++++++++++++++++++++------------------- test/netlib/http/test_request.py | 8 +- tox.ini | 2 +- 7 files changed, 114 insertions(+), 88 deletions(-) diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index 2998d259..cba96fbc 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -11,6 +11,7 @@ from mitmproxy import controller from mitmproxy import filt from netlib import wsgi from netlib import version +from netlib import strutils from netlib.http import cookies from netlib.http import http1 @@ -216,7 +217,7 @@ class ServerPlaybackState: self.nopop = nopop self.ignore_params = ignore_params self.ignore_content = ignore_content - self.ignore_payload_params = ignore_payload_params + self.ignore_payload_params = [strutils.always_bytes(x) for x in (ignore_payload_params or ())] self.ignore_host = ignore_host self.fmap = {} for i in flows: @@ -271,7 +272,7 @@ class ServerPlaybackState: v = r.headers.get(i) headers.append((i, v)) key.append(headers) - return hashlib.sha256(repr(key)).digest() + return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() def next_flow(self, request): """ diff --git a/netlib/http/message.py b/netlib/http/message.py index 0583c246..b268fec9 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -100,7 +100,10 @@ class Message(basetypes.Serializable): @content.setter def content(self, content): + # type: (Optional[bytes]) -> None self.data.content = content + if isinstance(content, six.text_type): + raise ValueError("Message content must be bytes, not {}".format(type(content).__name__)) if isinstance(content, bytes): self.headers["content-length"] = str(len(content)) diff --git a/netlib/http/request.py b/netlib/http/request.py index d9f4ed00..c4c39942 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -23,8 +23,20 @@ host_header_re = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") class RequestData(message.MessageData): def __init__(self, first_line_format, method, scheme, host, port, path, http_version, headers=(), content=None, timestamp_start=None, timestamp_end=None): + if isinstance(method, six.text_type): + method = method.encode("ascii", "strict") + if isinstance(scheme, six.text_type): + scheme = scheme.encode("ascii", "strict") + if isinstance(host, six.text_type): + host = host.encode("idna", "strict") + if isinstance(path, six.text_type): + path = path.encode("ascii", "strict") + if isinstance(http_version, six.text_type): + http_version = http_version.encode("ascii", "strict") if not isinstance(headers, nheaders.Headers): headers = nheaders.Headers(headers) + if isinstance(content, six.text_type): + raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) self.first_line_format = first_line_format self.method = method @@ -356,7 +368,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = netlib.http.url.encode(value) + self.content = netlib.http.url.encode(value).encode() @urlencoded_form.setter def urlencoded_form(self, value): diff --git a/netlib/http/response.py b/netlib/http/response.py index 17d69418..7cfb55c8 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function, division from email.utils import parsedate_tz, formatdate, mktime_tz import time +import six from netlib.http import cookies from netlib.http import headers as nheaders @@ -13,8 +14,14 @@ from netlib import human class ResponseData(message.MessageData): def __init__(self, http_version, status_code, reason=None, headers=(), content=None, timestamp_start=None, timestamp_end=None): + if isinstance(http_version, six.text_type): + http_version = http_version.encode("ascii", "strict") + if isinstance(reason, six.text_type): + reason = reason.encode("ascii", "strict") if not isinstance(headers, nheaders.Headers): headers = nheaders.Headers(headers) + if isinstance(content, six.text_type): + raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) self.http_version = http_version self.status_code = status_code diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 9eaab9aa..bf7622f6 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -1,7 +1,7 @@ import os.path -from six.moves import cStringIO as StringIO import mock +import io import netlib.utils from netlib.http import Headers @@ -65,7 +65,7 @@ class TestStickyCookieState: assert s.jar.keys() s, f = self._response("SSID=mooo", "www.google.com") - assert s.jar.keys()[0] == ('www.google.com', 80, '/') + assert list(s.jar.keys())[0] == ('www.google.com', 80, '/') # Test setting of multiple cookies c1 = "somecookie=test; Path=/" @@ -73,7 +73,7 @@ class TestStickyCookieState: s, f = self._response(c1, "www.google.com") f.response.headers["Set-Cookie"] = c2 s.handle_response(f) - googlekey = s.jar.keys()[0] + googlekey = list(s.jar.keys())[0] assert len(s.jar[googlekey].keys()) == 2 # Test setting of weird cookie keys @@ -88,8 +88,8 @@ class TestStickyCookieState: for c in cs: f.response.headers["Set-Cookie"] = c s.handle_response(f) - googlekey = s.jar.keys()[0] - assert len(s.jar[googlekey].keys()) == len(cs) + googlekey = list(s.jar.keys())[0] + assert len(s.jar[googlekey]) == len(cs) # Test overwriting of a cookie value c1 = "somecookie=helloworld; Path=/" @@ -97,12 +97,12 @@ class TestStickyCookieState: s, f = self._response(c1, "www.google.com") f.response.headers["Set-Cookie"] = c2 s.handle_response(f) - googlekey = s.jar.keys()[0] - assert len(s.jar[googlekey].keys()) == 1 - assert s.jar[googlekey]["somecookie"].items()[0][1] == "newvalue" + googlekey = list(s.jar.keys())[0] + assert len(s.jar[googlekey]) == 1 + assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue" def test_request(self): - s, f = self._response("SSID=mooo", "www.google.com") + s, f = self._response("SSID=mooo", b"www.google.com") assert "cookie" not in f.request.headers s.handle_request(f) assert "cookie" in f.request.headers @@ -264,26 +264,26 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = "paramx=x¶m1=1" + r.request.content = b"paramx=x¶m1=1" r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = "paramx=x¶m1=1" + r2.request.content = b"paramx=x¶m1=1" # same parameters assert s._hash(r) == s._hash(r2) # ignored parameters != - r2.request.content = "paramx=x¶m1=2" + r2.request.content = b"paramx=x¶m1=2" assert s._hash(r) == s._hash(r2) # missing parameter - r2.request.content = "paramx=x" + r2.request.content = b"paramx=x" assert s._hash(r) == s._hash(r2) # ignorable parameter added - r2.request.content = "paramx=x¶m1=2" + r2.request.content = b"paramx=x¶m1=2" assert s._hash(r) == s._hash(r2) # not ignorable parameter changed - r2.request.content = "paramx=y¶m1=1" + r2.request.content = b"paramx=y¶m1=1" assert not s._hash(r) == s._hash(r2) # not ignorable parameter missing - r2.request.content = "param1=1" + r2.request.content = b"param1=1" assert not s._hash(r) == s._hash(r2) def test_ignore_payload_params_other_content_type(self): @@ -292,14 +292,14 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/json" - r.request.content = '{"param1":"1"}' + r.request.content = b'{"param1":"1"}' r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/json" - r2.request.content = '{"param1":"1"}' + r2.request.content = b'{"param1":"1"}' # same content assert s._hash(r) == s._hash(r2) # distint content (note only x-www-form-urlencoded payload is analysed) - r2.request.content = '{"param1":"2"}' + r2.request.content = b'{"param1":"2"}' assert not s._hash(r) == s._hash(r2) def test_ignore_payload_wins_over_params(self): @@ -309,10 +309,10 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = "paramx=y" + r.request.content = b"paramx=y" r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = "paramx=x" + r2.request.content = b"paramx=x" # same parameters assert s._hash(r) == s._hash(r2) @@ -329,10 +329,10 @@ class TestServerPlaybackState: r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) - r.request.content = "foo" - r2.request.content = "foo" + r.request.content = b"foo" + r2.request.content = b"foo" assert s._hash(r) == s._hash(r2) - r2.request.content = "bar" + r2.request.content = b"bar" assert not s._hash(r) == s._hash(r2) # now ignoring content @@ -347,12 +347,12 @@ class TestServerPlaybackState: False) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) - r.request.content = "foo" - r2.request.content = "foo" + r.request.content = b"foo" + r2.request.content = b"foo" assert s._hash(r) == s._hash(r2) - r2.request.content = "bar" + r2.request.content = b"bar" assert s._hash(r) == s._hash(r2) - r2.request.content = "" + r2.request.content = b"" assert s._hash(r) == s._hash(r2) r2.request.content = None assert s._hash(r) == s._hash(r2) @@ -420,13 +420,13 @@ class TestFlow(object): def test_backup(self): f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp()) - f.request.content = "foo" + f.request.content = b"foo" assert not f.modified() f.backup() - f.request.content = "bar" + f.request.content = b"bar" assert f.modified() f.revert() - assert f.request.content == "foo" + assert f.request.content == b"foo" def test_backup_idempotence(self): f = tutils.tflow(resp=True) @@ -486,8 +486,8 @@ class TestFlow(object): def test_replace_unicode(self): f = tutils.tflow(resp=True) - f.response.content = "\xc2foo" - f.replace("foo", u"bar") + f.response.content = b"\xc2foo" + f.replace(b"foo", u"bar") def test_replace_no_content(self): f = tutils.tflow() @@ -497,34 +497,34 @@ class TestFlow(object): def test_replace(self): f = tutils.tflow(resp=True) f.request.headers["foo"] = "foo" - f.request.content = "afoob" + f.request.content = b"afoob" f.response.headers["foo"] = "foo" - f.response.content = "afoob" + f.response.content = b"afoob" assert f.replace("foo", "bar") == 6 assert f.request.headers["bar"] == "bar" - assert f.request.content == "abarb" + assert f.request.content == b"abarb" assert f.response.headers["bar"] == "bar" - assert f.response.content == "abarb" + assert f.response.content == b"abarb" def test_replace_encoded(self): f = tutils.tflow(resp=True) - f.request.content = "afoob" + f.request.content = b"afoob" f.request.encode("gzip") - f.response.content = "afoob" + f.response.content = b"afoob" f.response.encode("gzip") f.replace("foo", "bar") - assert f.request.content != "abarb" + assert f.request.content != b"abarb" f.request.decode() - assert f.request.content == "abarb" + assert f.request.content == b"abarb" - assert f.response.content != "abarb" + assert f.response.content != b"abarb" f.response.decode() - assert f.response.content == "abarb" + assert f.response.content == b"abarb" class TestState: @@ -667,7 +667,7 @@ class TestState: class TestSerialize: def _treader(self): - sio = StringIO() + sio = io.BytesIO() w = flow.FlowWriter(sio) for i in range(3): f = tutils.tflow(resp=True) @@ -684,9 +684,9 @@ class TestSerialize: return flow.FlowReader(sio) def test_roundtrip(self): - sio = StringIO() + sio = io.BytesIO() f = tutils.tflow() - f.request.content = "".join(chr(i) for i in range(255)) + f.request.content = bytes(bytearray(range(256))) w = flow.FlowWriter(sio) w.add(f) @@ -718,7 +718,7 @@ class TestSerialize: assert s.flows[0].request.host == "use-this-domain" def test_filter(self): - sio = StringIO() + sio = io.BytesIO() fl = filt.parse("~c 200") w = flow.FilteredFlowWriter(sio, fl) @@ -735,8 +735,8 @@ class TestSerialize: assert len(list(r.stream())) def test_error(self): - sio = StringIO() - sio.write("bogus") + sio = io.BytesIO() + sio.write(b"bogus") sio.seek(0) r = flow.FlowReader(sio) tutils.raises(FlowReadException, list, r.stream()) @@ -748,7 +748,7 @@ class TestSerialize: f = tutils.tflow() d = f.get_state() d["version"] = (0, 0) - sio = StringIO() + sio = io.BytesIO() tnetstring.dump(d, sio) sio.seek(0) @@ -938,7 +938,7 @@ class TestFlowMaster: None, False) r = tutils.tflow() - r.request.content = "gibble" + r.request.content = b"gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) @@ -1017,27 +1017,30 @@ class TestFlowMaster: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") - def r(): - r = flow.FlowReader(open(p, "rb")) - return list(r.stream()) + def read(): + with open(p, "rb") as f: + r = flow.FlowReader(f) + return list(r.stream()) s = flow.State() fm = flow.FlowMaster(None, s) f = tutils.tflow(resp=True) - fm.start_stream(file(p, "ab"), None) - fm.request(f) - fm.response(f) - fm.stop_stream() + with open(p, "ab") as tmpfile: + fm.start_stream(tmpfile, None) + fm.request(f) + fm.response(f) + fm.stop_stream() - assert r()[0].response + assert read()[0].response - f = tutils.tflow() - fm.start_stream(file(p, "ab"), None) - fm.request(f) - fm.shutdown() + with open(p, "ab") as tmpfile: + f = tutils.tflow() + fm.start_stream(tmpfile, None) + fm.request(f) + fm.shutdown() - assert not r()[1].response + assert not read()[1].response class TestRequest: @@ -1086,10 +1089,10 @@ class TestRequest: r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = "fOo" - r.content = "afoob" + r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 4 assert r.path == "path/boo" - assert "foo" not in r.content + assert b"foo" not in r.content assert r.headers["boo"] == "boo" def test_constrain_encoding(self): @@ -1108,9 +1111,9 @@ class TestRequest: r.headers["content-encoding"] = "identity" assert r.get_decoded_content() is None - r.content = "falafel" + r.content = b"falafel" r.encode("gzip") - assert r.get_decoded_content() == "falafel" + assert r.get_decoded_content() == b"falafel" def test_get_content_type(self): resp = HTTPResponse.wrap(netlib.tutils.tresp()) @@ -1129,9 +1132,9 @@ class TestResponse: def test_replace(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = "fOo" - r.content = "afoob" + r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 3 - assert "foo" not in r.content + assert b"foo" not in r.content assert r.headers["boo"] == "boo" def test_get_content_type(self): @@ -1208,24 +1211,24 @@ def test_replacehooks(): assert h.count() == 0 f = tutils.tflow() - f.request.content = "foo" + f.request.content = b"foo" h.add("~s", "foo", "bar") h.run(f) - assert f.request.content == "foo" + assert f.request.content == b"foo" f = tutils.tflow(resp=True) - f.request.content = "foo" - f.response.content = "foo" + f.request.content = b"foo" + f.response.content = b"foo" h.run(f) - assert f.response.content == "bar" - assert f.request.content == "foo" + assert f.response.content == b"bar" + assert f.request.content == b"foo" f = tutils.tflow() h.clear() h.add("~q", "foo", "bar") - f.request.content = "foo" + f.request.content = b"foo" h.run(f) - assert f.request.content == "bar" + assert f.request.content == b"bar" assert not h.add("~", "foo", "bar") assert not h.add("foo", "*", "bar") @@ -1257,10 +1260,10 @@ def test_setheaders(): assert h.count() == 0 f = tutils.tflow() - f.request.content = "foo" + f.request.content = b"foo" h.add("~s", "foo", "bar") h.run(f) - assert f.request.content == "foo" + assert f.request.content == b"foo" h.clear() h.add("~s", "one", "two") diff --git a/test/netlib/http/test_request.py b/test/netlib/http/test_request.py index c03db339..f3cd8b71 100644 --- a/test/netlib/http/test_request.py +++ b/test/netlib/http/test_request.py @@ -248,20 +248,20 @@ class TestRequestUtils(object): assert "gzip" in request.headers["Accept-Encoding"] def test_get_urlencoded_form(self): - request = treq(content="foobar=baz") + request = treq(content=b"foobar=baz") assert not request.urlencoded_form request.headers["Content-Type"] = "application/x-www-form-urlencoded" - assert list(request.urlencoded_form.items()) == [("foobar", "baz")] + assert list(request.urlencoded_form.items()) == [(b"foobar", b"baz")] def test_set_urlencoded_form(self): request = treq() - request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')] + request.urlencoded_form = [(b'foo', b'bar'), (b'rab', b'oof')] assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" assert request.content def test_get_multipart_form(self): - request = treq(content="foobar") + request = treq(content=b"foobar") assert not request.multipart_form request.headers["Content-Type"] = "multipart/form-data" diff --git a/tox.ini b/tox.ini index 4dbc67e4..6f189d35 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py test/mitmproxy/test_protocol_http2.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py test/mitmproxy/test_protocol_http2.py test/mitmproxy/test_flow.py HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From 76473d44e035306503a426ebcd55967798f243a1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 17:50:26 -0700 Subject: py3++ --- examples/iframe_injector.py | 2 +- examples/modify_response_body.py | 2 +- mitmproxy/flow/export.py | 5 +++-- test/mitmproxy/test_examples.py | 14 +++++++------- test/mitmproxy/test_flow_export.py | 10 +++++----- test/netlib/http/http1/test_assemble.py | 2 +- test/netlib/http/test_message.py | 4 ++-- 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py index 9495da93..ebb5fd02 100644 --- a/examples/iframe_injector.py +++ b/examples/iframe_injector.py @@ -24,5 +24,5 @@ def response(context, flow): height=0, width=0) html.body.insert(0, iframe) - flow.response.content = str(html) + flow.response.content = str(html).encode("utf8") context.log("Iframe inserted.") diff --git a/examples/modify_response_body.py b/examples/modify_response_body.py index 3034892e..994932a1 100644 --- a/examples/modify_response_body.py +++ b/examples/modify_response_body.py @@ -10,7 +10,7 @@ def start(context): raise ValueError('Usage: -s "modify_response_body.py old new"') # You may want to use Python's argparse for more sophisticated argument # parsing. - context.old, context.new = sys.argv[1], sys.argv[2] + context.old, context.new = sys.argv[1].encode(), sys.argv[2].encode() def response(context, flow): diff --git a/mitmproxy/flow/export.py b/mitmproxy/flow/export.py index 2b0f8984..67401719 100644 --- a/mitmproxy/flow/export.py +++ b/mitmproxy/flow/export.py @@ -75,7 +75,7 @@ def python_code(flow): data = "" if flow.request.body: - json_obj = is_json(flow.request.headers, flow.request.body) + json_obj = is_json(flow.request.headers, flow.request.content) if json_obj: data = "\njson = %s\n" % dictstr(sorted(json_obj.items()), " ") args += "\n json=json," @@ -100,11 +100,12 @@ def raw_request(flow): def is_json(headers, content): + # type: (netlib.http.Headers, bytes) -> bool if headers: ct = netlib.http.parse_content_type(headers.get("content-type", "")) if ct and "%s/%s" % (ct[0], ct[1]) == "application/json": try: - return json.loads(content) + return json.loads(content.decode("utf8", "surrogateescape")) except ValueError: return False return False diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 206a0366..f30973e7 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -84,11 +84,11 @@ def test_iframe_injector(): with example("iframe_injector.py") as ex: pass - flow = tutils.tflow(resp=netutils.tresp(content="mitmproxy")) + flow = tutils.tflow(resp=netutils.tresp(content=b"mitmproxy")) with example("iframe_injector.py http://example.org/evil_iframe") as ex: ex.run("response", flow) content = flow.response.content - assert 'iframe' in content and 'evil_iframe' in content + assert b'iframe' in content and b'evil_iframe' in content def test_modify_form(): @@ -96,11 +96,11 @@ def test_modify_form(): flow = tutils.tflow(req=netutils.treq(headers=form_header)) with example("modify_form.py") as ex: ex.run("request", flow) - assert flow.request.urlencoded_form["mitmproxy"] == "rocks" + assert flow.request.urlencoded_form[b"mitmproxy"] == b"rocks" flow.request.headers["content-type"] = "" ex.run("request", flow) - assert list(flow.request.urlencoded_form.items()) == [("foo", "bar")] + assert list(flow.request.urlencoded_form.items()) == [(b"foo", b"bar")] def test_modify_querystring(): @@ -119,11 +119,11 @@ def test_modify_response_body(): with example("modify_response_body.py"): assert True - flow = tutils.tflow(resp=netutils.tresp(content="I <3 mitmproxy")) + flow = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) with example("modify_response_body.py mitmproxy rocks") as ex: - assert ex.ctx.old == "mitmproxy" and ex.ctx.new == "rocks" + assert ex.ctx.old == b"mitmproxy" and ex.ctx.new == b"rocks" ex.run("response", flow) - assert flow.response.content == "I <3 rocks" + assert flow.response.content == b"I <3 rocks" def test_redirect_requests(): diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 33c5137a..e6d65e40 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -60,7 +60,7 @@ class TestExportPythonCode(): def test_post_json(self): p = req_post() - p.content = '{"name": "example", "email": "example@example.com"}' + p.content = b'{"name": "example", "email": "example@example.com"}' p.headers = Headers(content_type="application/json") flow = tutils.tflow(req=p) python_equals("data/test_flow_export/python_post_json.py", export.python_code(flow)) @@ -112,7 +112,7 @@ class TestExportLocustCode(): def test_post(self): p = req_post() - p.content = '''content''' + p.content = b'content' p.headers = '' flow = tutils.tflow(req=p) python_equals("data/test_flow_export/locust_post.py", export.locust_code(flow)) @@ -142,14 +142,14 @@ class TestIsJson(): def test_json_type(self): headers = Headers(content_type="application/json") - assert export.is_json(headers, "foobar") is False + assert export.is_json(headers, b"foobar") is False def test_valid(self): headers = Headers(content_type="application/foobar") - j = export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + j = export.is_json(headers, b'{"name": "example", "email": "example@example.com"}') assert j is False def test_valid2(self): headers = Headers(content_type="application/json") - j = export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + j = export.is_json(headers, b'{"name": "example", "email": "example@example.com"}') assert isinstance(j, dict) diff --git a/test/netlib/http/http1/test_assemble.py b/test/netlib/http/http1/test_assemble.py index 50d29384..841ea58a 100644 --- a/test/netlib/http/http1/test_assemble.py +++ b/test/netlib/http/http1/test_assemble.py @@ -24,7 +24,7 @@ def test_assemble_request(): def test_assemble_request_head(): - c = assemble_request_head(treq(content="foo")) + c = assemble_request_head(treq(content=b"foo")) assert b"GET" in c assert b"qvalue" in c assert b"content-length" in c diff --git a/test/netlib/http/test_message.py b/test/netlib/http/test_message.py index f5bf7f0c..ab2ac628 100644 --- a/test/netlib/http/test_message.py +++ b/test/netlib/http/test_message.py @@ -7,8 +7,8 @@ from netlib.tutils import tresp def _test_passthrough_attr(message, attr): assert getattr(message, attr) == getattr(message.data, attr) - setattr(message, attr, "foo") - assert getattr(message.data, attr) == "foo" + setattr(message, attr, b"foo") + assert getattr(message.data, attr) == b"foo" def _test_decoded_attr(message, attr): -- cgit v1.2.3 From 00dce240150595e41719e512f1b156103c3c9c31 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 18:02:59 -0700 Subject: tests++ --- netlib/strutils.py | 2 +- test/netlib/test_strutils.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/netlib/strutils.py b/netlib/strutils.py index cda70651..7d49e56e 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -119,7 +119,7 @@ def is_mostly_bin(s): return sum( i < 9 or 13 < i < 32 or 126 < i for i in six.iterbytes(s[:100]) - ) > 30 + ) / len(s[:100]) > 0.3 def is_xml(s): diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index f88e33ed..68bfdb94 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -3,6 +3,13 @@ import six from netlib import strutils, tutils +def test_always_bytes(): + assert strutils.always_bytes(bytes(bytearray(range(256)))) == bytes(bytearray(range(256))) + assert strutils.always_bytes("foo") == b"foo" + with tutils.raises(ValueError): + strutils.always_bytes(u"\u2605", "ascii") + + def test_native(): with tutils.raises(TypeError): strutils.native(42) @@ -31,6 +38,9 @@ def test_escape_control_characters(): u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' ) + with tutils.raises(ValueError): + strutils.escape_control_characters(b"foo") + def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" @@ -68,6 +78,11 @@ def test_escaped_str_to_bytes(): strutils.escaped_str_to_bytes(b"very byte") +def test_is_mostly_bin(): + assert not strutils.is_mostly_bin(b"foo\xFF") + assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10) + + def test_is_xml(): assert not strutils.is_xml(b"foo") assert strutils.is_xml(b" Date: Thu, 7 Jul 2016 19:34:57 -0700 Subject: fix tests --- netlib/strutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/netlib/strutils.py b/netlib/strutils.py index 7d49e56e..9208f954 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, print_function, division import re import codecs -- cgit v1.2.3 From c048ae1d5b652ad4778917e624ace217e1ecfd91 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 18:37:33 -0700 Subject: remove context from all scripts --- examples/add_header.py | 2 +- examples/change_upstream_proxy.py | 2 +- examples/custom_contentviews.py | 8 ++--- examples/dns_spoofing.py | 2 +- examples/dup_and_replay.py | 9 +++-- examples/fail_with_500.py | 2 +- examples/filt.py | 10 +++--- examples/flowwriter.py | 10 +++--- examples/har_extractor.py | 12 +++++-- examples/iframe_injector.py | 14 ++++---- examples/modify_form.py | 2 +- examples/modify_querystring.py | 2 +- examples/modify_response_body.py | 14 +++++--- examples/nonblocking.py | 7 ++-- examples/proxapp.py | 7 ++-- examples/redirect_requests.py | 2 +- examples/sslstrip.py | 14 ++++---- examples/stream.py | 2 +- examples/stream_modify.py | 2 +- examples/stub.py | 41 +++++++++++----------- examples/tcp_message.py | 2 +- examples/tls_passthrough.py | 24 +++++++------ examples/upsidedownternet.py | 2 +- test/mitmproxy/data/scripts/a.py | 4 +-- test/mitmproxy/data/scripts/all.py | 29 +++++++-------- .../mitmproxy/data/scripts/concurrent_decorator.py | 2 +- .../data/scripts/concurrent_decorator_err.py | 2 +- test/mitmproxy/data/scripts/duplicate_flow.py | 8 +++-- test/mitmproxy/data/scripts/reqerr.py | 4 +-- test/mitmproxy/data/scripts/starterr.py | 2 +- test/mitmproxy/data/scripts/stream_modify.py | 2 +- test/mitmproxy/data/scripts/tcp_stream_modify.py | 2 +- test/mitmproxy/data/scripts/unloaderr.py | 2 +- 33 files changed, 137 insertions(+), 112 deletions(-) diff --git a/examples/add_header.py b/examples/add_header.py index cf1b53cc..3e0b5f1e 100644 --- a/examples/add_header.py +++ b/examples/add_header.py @@ -1,2 +1,2 @@ -def response(context, flow): +def response(flow): flow.response.headers["newheader"] = "foo" diff --git a/examples/change_upstream_proxy.py b/examples/change_upstream_proxy.py index 34a6eece..49d5379f 100644 --- a/examples/change_upstream_proxy.py +++ b/examples/change_upstream_proxy.py @@ -14,7 +14,7 @@ def proxy_address(flow): return ("localhost", 8081) -def request(context, flow): +def request(flow): if flow.request.method == "CONNECT": # If the decision is done by domain, one could also modify the server address here. # We do it after CONNECT here to have the request data available as well. diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py index 92fb6a58..5a63e2a0 100644 --- a/examples/custom_contentviews.py +++ b/examples/custom_contentviews.py @@ -62,9 +62,9 @@ class ViewPigLatin(contentviews.View): pig_view = ViewPigLatin() -def start(context): - context.add_contentview(pig_view) +def start(): + contentviews.add(pig_view) -def done(context): - context.remove_contentview(pig_view) +def done(): + contentviews.remove(pig_view) diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py index 8d715f33..c020047f 100644 --- a/examples/dns_spoofing.py +++ b/examples/dns_spoofing.py @@ -28,7 +28,7 @@ import re parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") -def request(context, flow): +def request(flow): if flow.client_conn.ssl_established: flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() diff --git a/examples/dup_and_replay.py b/examples/dup_and_replay.py index 9ba91d3b..b47bf951 100644 --- a/examples/dup_and_replay.py +++ b/examples/dup_and_replay.py @@ -1,4 +1,7 @@ -def request(context, flow): - f = context.duplicate_flow(flow) +from mitmproxy import master + + +def request(flow): + f = master.duplicate_flow(flow) f.request.path = "/changed" - context.replay_request(f) + master.replay_request(f, block=True, run_scripthooks=False) diff --git a/examples/fail_with_500.py b/examples/fail_with_500.py index aec85b50..9710f74a 100644 --- a/examples/fail_with_500.py +++ b/examples/fail_with_500.py @@ -1,3 +1,3 @@ -def response(context, flow): +def response(flow): flow.response.status_code = 500 flow.response.content = b"" diff --git a/examples/filt.py b/examples/filt.py index 1a423845..21744edd 100644 --- a/examples/filt.py +++ b/examples/filt.py @@ -3,14 +3,16 @@ import sys from mitmproxy import filt +state = {} -def start(context): + +def start(): if len(sys.argv) != 2: raise ValueError("Usage: -s 'filt.py FILTER'") - context.filter = filt.parse(sys.argv[1]) + state["filter"] = filt.parse(sys.argv[1]) -def response(context, flow): - if flow.match(context.filter): +def response(flow): + if flow.match(state["filter"]): print("Flow matches filter:") print(flow) diff --git a/examples/flowwriter.py b/examples/flowwriter.py index cb5ccb0d..07c7ca20 100644 --- a/examples/flowwriter.py +++ b/examples/flowwriter.py @@ -3,8 +3,10 @@ import sys from mitmproxy.flow import FlowWriter +state = {} -def start(context): + +def start(): if len(sys.argv) != 2: raise ValueError('Usage: -s "flowriter.py filename"') @@ -12,9 +14,9 @@ def start(context): f = sys.stdout else: f = open(sys.argv[1], "wb") - context.flow_writer = FlowWriter(f) + state["flow_writer"] = FlowWriter(f) -def response(context, flow): +def response(flow): if random.choice([True, False]): - context.flow_writer.add(flow) + state["flow_writer"].add(flow) diff --git a/examples/har_extractor.py b/examples/har_extractor.py index a5c05519..b136bd40 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -54,7 +54,13 @@ class _HARLog(HAR.log): return self.__page_list__ -def start(context): +class Context(object): + pass + +context = Context() + + +def start(): """ On start we create a HARLog instance. You will have to adapt this to suit your actual needs of HAR generation. As it will probably be @@ -79,7 +85,7 @@ def start(context): context.seen_server = set() -def response(context, flow): +def response(flow): """ Called when a server response has been received. At the time of this message both a request and a response are present and completely done. @@ -201,7 +207,7 @@ def response(context, flow): context.HARLog.add(entry) -def done(context): +def done(): """ Called once on script shutdown, after any other events. """ diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py index ebb5fd02..70247d31 100644 --- a/examples/iframe_injector.py +++ b/examples/iframe_injector.py @@ -4,25 +4,27 @@ import sys from bs4 import BeautifulSoup from mitmproxy.models import decoded +iframe_url = None -def start(context): + +def start(): if len(sys.argv) != 2: raise ValueError('Usage: -s "iframe_injector.py url"') - context.iframe_url = sys.argv[1] + global iframe_url + iframe_url = sys.argv[1] -def response(context, flow): - if flow.request.host in context.iframe_url: +def response(flow): + if flow.request.host in iframe_url: return with decoded(flow.response): # Remove content encoding (gzip, ...) html = BeautifulSoup(flow.response.content, "lxml") if html.body: iframe = html.new_tag( "iframe", - src=context.iframe_url, + src=iframe_url, frameborder=0, height=0, width=0) html.body.insert(0, iframe) flow.response.content = str(html).encode("utf8") - context.log("Iframe inserted.") diff --git a/examples/modify_form.py b/examples/modify_form.py index 3fe0cf96..b63a1586 100644 --- a/examples/modify_form.py +++ b/examples/modify_form.py @@ -1,4 +1,4 @@ -def request(context, flow): +def request(flow): if flow.request.urlencoded_form: flow.request.urlencoded_form["mitmproxy"] = "rocks" else: diff --git a/examples/modify_querystring.py b/examples/modify_querystring.py index b89e5c8d..ee8a89ad 100644 --- a/examples/modify_querystring.py +++ b/examples/modify_querystring.py @@ -1,2 +1,2 @@ -def request(context, flow): +def request(flow): flow.request.query["mitmproxy"] = "rocks" diff --git a/examples/modify_response_body.py b/examples/modify_response_body.py index 994932a1..23ad0151 100644 --- a/examples/modify_response_body.py +++ b/examples/modify_response_body.py @@ -5,16 +5,20 @@ import sys from mitmproxy.models import decoded -def start(context): +state = {} + + +def start(): if len(sys.argv) != 3: raise ValueError('Usage: -s "modify_response_body.py old new"') # You may want to use Python's argparse for more sophisticated argument # parsing. - context.old, context.new = sys.argv[1].encode(), sys.argv[2].encode() + state["old"], state["new"] = sys.argv[1].encode(), sys.argv[2].encode() -def response(context, flow): +def response(flow): with decoded(flow.response): # automatically decode gzipped responses. flow.response.content = flow.response.content.replace( - context.old, - context.new) + state["old"], + state["new"] + ) diff --git a/examples/nonblocking.py b/examples/nonblocking.py index 4609f389..05a26921 100644 --- a/examples/nonblocking.py +++ b/examples/nonblocking.py @@ -1,9 +1,10 @@ import time +import mitmproxy from mitmproxy.script import concurrent @concurrent # Remove this and see what happens -def request(context, flow): - context.log("handle request: %s%s" % (flow.request.host, flow.request.path)) +def request(flow): + mitmproxy.log("handle request: %s%s" % (flow.request.host, flow.request.path)) time.sleep(5) - context.log("start request: %s%s" % (flow.request.host, flow.request.path)) + mitmproxy.log("start request: %s%s" % (flow.request.host, flow.request.path)) diff --git a/examples/proxapp.py b/examples/proxapp.py index 613d3f8b..ddc38544 100644 --- a/examples/proxapp.py +++ b/examples/proxapp.py @@ -4,6 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose a single simplest-possible page. """ from flask import Flask +import mitmproxy app = Flask("proxapp") @@ -15,10 +16,10 @@ def hello_world(): # Register the app using the magic domain "proxapp" on port 80. Requests to # this domain and port combination will now be routed to the WSGI app instance. -def start(context): - context.app_registry.add(app, "proxapp", 80) +def start(): + mitmproxy.master.apps.add(app, "proxapp", 80) # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set) # but won't send any data. - context.app_registry.add(app, "example.com", 443) + mitmproxy.master.apps.add(app, "example.com", 443) diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index af2aa907..36594bcd 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -5,7 +5,7 @@ from mitmproxy.models import HTTPResponse from netlib.http import Headers -def request(context, flow): +def request(flow): # pretty_host takes the "Host" header of the request into account, # which is useful in transparent mode where we usually only have the IP # otherwise. diff --git a/examples/sslstrip.py b/examples/sslstrip.py index 8dde8e3e..afc95fc8 100644 --- a/examples/sslstrip.py +++ b/examples/sslstrip.py @@ -2,23 +2,21 @@ from netlib.http import decoded import re from six.moves import urllib +# set of SSL/TLS capable hosts +secure_hosts = set() -def start(context): - # set of SSL/TLS capable hosts - context.secure_hosts = set() - -def request(context, flow): +def request(flow): flow.request.headers.pop('If-Modified-Since', None) flow.request.headers.pop('Cache-Control', None) # proxy connections to SSL-enabled hosts - if flow.request.pretty_host in context.secure_hosts: + if flow.request.pretty_host in secure_hosts: flow.request.scheme = 'https' flow.request.port = 443 -def response(context, flow): +def response(flow): with decoded(flow.response): flow.request.headers.pop('Strict-Transport-Security', None) flow.request.headers.pop('Public-Key-Pins', None) @@ -31,7 +29,7 @@ def response(context, flow): location = flow.response.headers['Location'] hostname = urllib.parse.urlparse(location).hostname if hostname: - context.secure_hosts.add(hostname) + secure_hosts.add(hostname) flow.response.headers['Location'] = location.replace('https://', 'http://', 1) # strip secure flag from 'Set-Cookie' headers diff --git a/examples/stream.py b/examples/stream.py index 3adbe437..8598f329 100644 --- a/examples/stream.py +++ b/examples/stream.py @@ -1,4 +1,4 @@ -def responseheaders(context, flow): +def responseheaders(flow): """ Enables streaming for all responses. """ diff --git a/examples/stream_modify.py b/examples/stream_modify.py index aa395c03..5e5da95b 100644 --- a/examples/stream_modify.py +++ b/examples/stream_modify.py @@ -16,5 +16,5 @@ def modify(chunks): yield chunk.replace("foo", "bar") -def responseheaders(context, flow): +def responseheaders(flow): flow.response.stream = modify diff --git a/examples/stub.py b/examples/stub.py index a0f73538..a4f16699 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -1,79 +1,80 @@ +import mitmproxy """ This is a script stub, with definitions for all events. """ -def start(context): +def start(): """ Called once on script startup, before any other events. """ - context.log("start") + mitmproxy.log("start") -def clientconnect(context, root_layer): +def clientconnect(root_layer): """ Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests """ - context.log("clientconnect") + mitmproxy.log("clientconnect") -def request(context, flow): +def request(flow): """ Called when a client request has been received. """ - context.log("request") + mitmproxy.log("request") -def serverconnect(context, server_conn): +def serverconnect(server_conn): """ Called when the proxy initiates a connection to the target server. Note that a connection can correspond to multiple HTTP requests """ - context.log("serverconnect") + mitmproxy.log("serverconnect") -def responseheaders(context, flow): +def responseheaders(flow): """ Called when the response headers for a server response have been received, but the response body has not been processed yet. Can be used to tell mitmproxy to stream the response. """ - context.log("responseheaders") + mitmproxy.log("responseheaders") -def response(context, flow): +def response(flow): """ Called when a server response has been received. """ - context.log("response") + mitmproxy.log("response") -def error(context, flow): +def error(flow): """ Called when a flow error has occured, e.g. invalid server responses, or interrupted connections. This is distinct from a valid server HTTP error response, which is simply a response with an HTTP error code. """ - context.log("error") + mitmproxy.log("error") -def serverdisconnect(context, server_conn): +def serverdisconnect(server_conn): """ Called when the proxy closes the connection to the target server. """ - context.log("serverdisconnect") + mitmproxy.log("serverdisconnect") -def clientdisconnect(context, root_layer): +def clientdisconnect(root_layer): """ Called when a client disconnects from the proxy. """ - context.log("clientdisconnect") + mitmproxy.log("clientdisconnect") -def done(context): +def done(): """ Called once on script shutdown, after any other events. """ - context.log("done") + mitmproxy.log("done") diff --git a/examples/tcp_message.py b/examples/tcp_message.py index 6eced0dc..b431c23f 100644 --- a/examples/tcp_message.py +++ b/examples/tcp_message.py @@ -11,7 +11,7 @@ mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py from netlib import strutils -def tcp_message(ctx, tcp_msg): +def tcp_message(tcp_msg): modified_msg = tcp_msg.message.replace("foo", "bar") is_modified = False if modified_msg == tcp_msg.message else True diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py index 50aab65b..374020e7 100644 --- a/examples/tls_passthrough.py +++ b/examples/tls_passthrough.py @@ -20,13 +20,14 @@ Example: Authors: Maximilian Hils, Matthew Tuusberg """ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division import collections import random import sys from enum import Enum +import mitmproxy from mitmproxy.exceptions import TlsProtocolException from mitmproxy.protocol import TlsLayer, RawTCPLayer @@ -97,7 +98,6 @@ class TlsFeedback(TlsLayer): def _establish_tls_with_client(self): server_address = self.server_conn.address - tls_strategy = self.script_context.tls_strategy try: super(TlsFeedback, self)._establish_tls_with_client() @@ -110,15 +110,18 @@ class TlsFeedback(TlsLayer): # inline script hooks below. +tls_strategy = None -def start(context): + +def start(): + global tls_strategy if len(sys.argv) == 2: - context.tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) + tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) else: - context.tls_strategy = ConservativeStrategy() + tls_strategy = ConservativeStrategy() -def next_layer(context, next_layer): +def next_layer(next_layer): """ This hook does the actual magic - if the next layer is planned to be a TLS layer, we check if we want to enter pass-through mode instead. @@ -126,14 +129,13 @@ def next_layer(context, next_layer): if isinstance(next_layer, TlsLayer) and next_layer._client_tls: server_address = next_layer.server_conn.address - if context.tls_strategy.should_intercept(server_address): + if tls_strategy.should_intercept(server_address): # We try to intercept. # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked. next_layer.__class__ = TlsFeedback - next_layer.script_context = context else: # We don't intercept - reply with a pass-through layer and add a "skipped" entry. - context.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") - next_layer_replacement = RawTCPLayer(next_layer.ctx, logging=False) + mitmproxy.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) next_layer.reply.send(next_layer_replacement) - context.tls_strategy.record_skipped(server_address) + tls_strategy.record_skipped(server_address) diff --git a/examples/upsidedownternet.py b/examples/upsidedownternet.py index 9aac9f05..fafdefce 100644 --- a/examples/upsidedownternet.py +++ b/examples/upsidedownternet.py @@ -3,7 +3,7 @@ from PIL import Image from mitmproxy.models import decoded -def response(context, flow): +def response(flow): if flow.response.headers.get("content-type", "").startswith("image"): with decoded(flow.response): # automatically decode gzipped responses. try: diff --git a/test/mitmproxy/data/scripts/a.py b/test/mitmproxy/data/scripts/a.py index 33dbaa64..ab0dbf96 100644 --- a/test/mitmproxy/data/scripts/a.py +++ b/test/mitmproxy/data/scripts/a.py @@ -5,12 +5,12 @@ from a_helper import parser var = 0 -def start(ctx): +def start(): global var var = parser.parse_args(sys.argv[1:]).var -def here(ctx): +def here(): global var var += 1 return var diff --git a/test/mitmproxy/data/scripts/all.py b/test/mitmproxy/data/scripts/all.py index dad2aade..17ffe33f 100644 --- a/test/mitmproxy/data/scripts/all.py +++ b/test/mitmproxy/data/scripts/all.py @@ -1,36 +1,37 @@ +import mitmproxy log = [] -def clientconnect(ctx, cc): - ctx.log("XCLIENTCONNECT") +def clientconnect(cc): + mitmproxy.log("XCLIENTCONNECT") log.append("clientconnect") -def serverconnect(ctx, cc): - ctx.log("XSERVERCONNECT") +def serverconnect(cc): + mitmproxy.log("XSERVERCONNECT") log.append("serverconnect") -def request(ctx, f): - ctx.log("XREQUEST") +def request(f): + mitmproxy.log("XREQUEST") log.append("request") -def response(ctx, f): - ctx.log("XRESPONSE") +def response(f): + mitmproxy.log("XRESPONSE") log.append("response") -def responseheaders(ctx, f): - ctx.log("XRESPONSEHEADERS") +def responseheaders(f): + mitmproxy.log("XRESPONSEHEADERS") log.append("responseheaders") -def clientdisconnect(ctx, cc): - ctx.log("XCLIENTDISCONNECT") +def clientdisconnect(cc): + mitmproxy.log("XCLIENTDISCONNECT") log.append("clientdisconnect") -def error(ctx, cc): - ctx.log("XERROR") +def error(cc): + mitmproxy.log("XERROR") log.append("error") diff --git a/test/mitmproxy/data/scripts/concurrent_decorator.py b/test/mitmproxy/data/scripts/concurrent_decorator.py index e017f605..162c00f4 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator.py +++ b/test/mitmproxy/data/scripts/concurrent_decorator.py @@ -3,5 +3,5 @@ from mitmproxy.script import concurrent @concurrent -def request(context, flow): +def request(flow): time.sleep(0.1) diff --git a/test/mitmproxy/data/scripts/concurrent_decorator_err.py b/test/mitmproxy/data/scripts/concurrent_decorator_err.py index 349e5dd6..756869c8 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator_err.py +++ b/test/mitmproxy/data/scripts/concurrent_decorator_err.py @@ -2,5 +2,5 @@ from mitmproxy.script import concurrent @concurrent -def start(context): +def start(): pass diff --git a/test/mitmproxy/data/scripts/duplicate_flow.py b/test/mitmproxy/data/scripts/duplicate_flow.py index e13af786..a50d2cb5 100644 --- a/test/mitmproxy/data/scripts/duplicate_flow.py +++ b/test/mitmproxy/data/scripts/duplicate_flow.py @@ -1,4 +1,6 @@ +import mitmproxy -def request(ctx, f): - f = ctx.duplicate_flow(f) - ctx.replay_request(f) + +def request(f): + f = mitmproxy.master.duplicate_flow(f) + mitmproxy.master.replay_request(f, block=True, run_scripthooks=False) diff --git a/test/mitmproxy/data/scripts/reqerr.py b/test/mitmproxy/data/scripts/reqerr.py index e7c503a8..7b419361 100644 --- a/test/mitmproxy/data/scripts/reqerr.py +++ b/test/mitmproxy/data/scripts/reqerr.py @@ -1,2 +1,2 @@ -def request(ctx, r): - raise ValueError +def request(r): + raise ValueError() diff --git a/test/mitmproxy/data/scripts/starterr.py b/test/mitmproxy/data/scripts/starterr.py index 82d773bd..28ba2ff1 100644 --- a/test/mitmproxy/data/scripts/starterr.py +++ b/test/mitmproxy/data/scripts/starterr.py @@ -1,3 +1,3 @@ -def start(ctx): +def start(): raise ValueError() diff --git a/test/mitmproxy/data/scripts/stream_modify.py b/test/mitmproxy/data/scripts/stream_modify.py index 8221b0dd..4fbf45c2 100644 --- a/test/mitmproxy/data/scripts/stream_modify.py +++ b/test/mitmproxy/data/scripts/stream_modify.py @@ -3,5 +3,5 @@ def modify(chunks): yield chunk.replace(b"foo", b"bar") -def responseheaders(context, flow): +def responseheaders(flow): flow.response.stream = modify diff --git a/test/mitmproxy/data/scripts/tcp_stream_modify.py b/test/mitmproxy/data/scripts/tcp_stream_modify.py index 0965beba..2281e6e6 100644 --- a/test/mitmproxy/data/scripts/tcp_stream_modify.py +++ b/test/mitmproxy/data/scripts/tcp_stream_modify.py @@ -1,4 +1,4 @@ -def tcp_message(ctx, flow): +def tcp_message(flow): message = flow.messages[-1] if not message.from_client: message.content = message.content.replace(b"foo", b"bar") diff --git a/test/mitmproxy/data/scripts/unloaderr.py b/test/mitmproxy/data/scripts/unloaderr.py index fba02734..6a48ab43 100644 --- a/test/mitmproxy/data/scripts/unloaderr.py +++ b/test/mitmproxy/data/scripts/unloaderr.py @@ -1,2 +1,2 @@ -def done(ctx): +def done(): raise RuntimeError() -- cgit v1.2.3 From 7c67faa8da39f428d1860bccae806137943b66a6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 23:50:55 -0700 Subject: remove script contexts --- docs/scripting/inlinescripts.rst | 23 ++-------- examples/har_extractor.py | 7 ++-- mitmproxy/__init__.py | 5 +++ mitmproxy/console/master.py | 6 +-- mitmproxy/controller.py | 46 +++++++++++++------- mitmproxy/flow/master.py | 15 ++++--- mitmproxy/script/__init__.py | 2 - mitmproxy/script/concurrent.py | 4 +- mitmproxy/script/reloader.py | 4 +- mitmproxy/script/script.py | 72 ++++++++++++++------------------ mitmproxy/script/script_context.py | 61 --------------------------- test/mitmproxy/script/test_concurrent.py | 4 +- test/mitmproxy/script/test_reloader.py | 2 +- test/mitmproxy/script/test_script.py | 28 ++++++------- test/mitmproxy/test_examples.py | 49 +++++++++------------- test/mitmproxy/test_server.py | 3 +- 16 files changed, 131 insertions(+), 200 deletions(-) delete mode 100644 mitmproxy/script/script_context.py diff --git a/docs/scripting/inlinescripts.rst b/docs/scripting/inlinescripts.rst index 1ee44972..bc9d5ff5 100644 --- a/docs/scripting/inlinescripts.rst +++ b/docs/scripting/inlinescripts.rst @@ -15,9 +15,7 @@ client: :caption: examples/add_header.py :language: python -The first argument to each event method is an instance of -:py:class:`~mitmproxy.script.ScriptContext` that lets the script interact with the global mitmproxy -state. The **response** event also gets an instance of :py:class:`~mitmproxy.models.HTTPFlow`, +All events that deal with an HTTP request get an instance of :py:class:`~mitmproxy.models.HTTPFlow`, which we can use to manipulate the response itself. We can now run this script using mitmdump or mitmproxy as follows: @@ -36,11 +34,6 @@ We encourage you to either browse them locally or on `GitHub`_. Events ------ -The ``context`` argument passed to each event method is always a -:py:class:`~mitmproxy.script.ScriptContext` instance. It is guaranteed to be the same object -for the scripts lifetime and is not shared between multiple inline scripts. You can safely use it -to store any form of state you require. - Script Lifecycle Events ^^^^^^^^^^^^^^^^^^^^^^^ @@ -155,8 +148,9 @@ The canonical API documentation is the code, which you can browse here, locally The main classes you will deal with in writing mitmproxy scripts are: -:py:class:`~mitmproxy.script.ScriptContext` - - A handle for interacting with mitmproxy's Flow Master from within scripts. +:py:class:`mitmproxy.flow.FlowMaster` + - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or + :py:class:`mitmproxy.console.ConsoleMaster`. :py:class:`~mitmproxy.models.ClientConnection` - Describes a client connection. :py:class:`~mitmproxy.models.ServerConnection` @@ -173,16 +167,7 @@ The main classes you will deal with in writing mitmproxy scripts are: - A dictionary-like object for managing HTTP headers. :py:class:`netlib.certutils.SSLCert` - Exposes information SSL certificates. -:py:class:`mitmproxy.flow.FlowMaster` - - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or - :py:class:`mitmproxy.console.ConsoleMaster`. - -Script Context --------------- -.. autoclass:: mitmproxy.script.ScriptContext - :members: - :undoc-members: Running scripts in parallel --------------------------- diff --git a/examples/har_extractor.py b/examples/har_extractor.py index b136bd40..208a2fa8 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -2,6 +2,7 @@ This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser to generate a HAR log object. """ +import mitmproxy import six import sys import pytz @@ -218,17 +219,17 @@ def done(): compressed_json_dump = context.HARLog.compress() if context.dump_file == '-': - context.log(pprint.pformat(json.loads(json_dump))) + mitmproxy.log(pprint.pformat(json.loads(json_dump))) elif context.dump_file.endswith('.zhar'): file(context.dump_file, "w").write(compressed_json_dump) else: file(context.dump_file, "w").write(json_dump) - context.log( + mitmproxy.log( "HAR log finished with %s bytes (%s bytes compressed)" % ( len(json_dump), len(compressed_json_dump) ) ) - context.log( + mitmproxy.log( "Compression rate is %s%%" % str( 100. * len(compressed_json_dump) / len(json_dump) ) diff --git a/mitmproxy/__init__.py b/mitmproxy/__init__.py index e69de29b..99ce7c9c 100644 --- a/mitmproxy/__init__.py +++ b/mitmproxy/__init__.py @@ -0,0 +1,5 @@ +from typing import Callable # noqa +from mitmproxy import flow # noqa + +master = None # type: flow.FlowMaster +log = None # type: Callable[[str], None] diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 95c9704d..93b5766d 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -366,7 +366,7 @@ class ConsoleMaster(flow.FlowMaster): signals.add_event("Running script on flow: %s" % command, "debug") try: - s = script.Script(command, script.ScriptContext(self)) + s = script.Script(command) s.load() except script.ScriptException as e: signals.status_message.send( @@ -812,6 +812,6 @@ class ConsoleMaster(flow.FlowMaster): @controller.handler def script_change(self, script): if super(ConsoleMaster, self).script_change(script): - signals.status_message.send(message='"{}" reloaded.'.format(script.filename)) + signals.status_message.send(message='"{}" reloaded.'.format(script.path)) else: - signals.status_message.send(message='Error reloading "{}".'.format(script.filename)) + signals.status_message.send(message='Error reloading "{}".'.format(script.path)) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index a170d868..222ebc69 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -2,11 +2,12 @@ from __future__ import absolute_import, print_function, division import functools import threading +import contextlib from six.moves import queue +import mitmproxy from netlib import basethread - from . import exceptions @@ -34,6 +35,16 @@ Events = frozenset([ ]) +class Log(object): + def __init__(self, master): + self.master = master + + def __call__(self, text, level="info"): + self.master.add_event(text, level) + + # We may want to add .log(), .warn() etc. here at a later point in time + + class Master(object): """ The master handles mitmproxy's main event loop. @@ -45,6 +56,20 @@ class Master(object): for i in servers: self.add_server(i) + @contextlib.contextmanager + def handlecontext(self): + # Handlecontexts also have to nest - leave cleanup to the outermost + if mitmproxy.master: + yield + return + mitmproxy.master = self + mitmproxy.log = Log(self) + try: + yield + finally: + mitmproxy.master = None + mitmproxy.log = None + def add_server(self, server): # We give a Channel to the server which can be used to communicate with the master channel = Channel(self.event_queue, self.should_exit) @@ -77,8 +102,8 @@ class Master(object): if mtype not in Events: raise exceptions.ControlException("Unknown event %s" % repr(mtype)) handle_func = getattr(self, mtype) - if not hasattr(handle_func, "__dict__"): - raise exceptions.ControlException("Handler %s not a function" % mtype) + if not callable(handle_func): + raise exceptions.ControlException("Handler %s not callable" % mtype) if not handle_func.__dict__.get("__handler"): raise exceptions.ControlException( "Handler function %s is not decorated with controller.handler" % ( @@ -151,15 +176,7 @@ class Channel(object): def handler(f): @functools.wraps(f) - def wrapper(*args, **kwargs): - # We can either be called as a method, or as a wrapped solo function - if len(args) == 1: - message = args[0] - elif len(args) == 2: - message = args[1] - else: - raise exceptions.ControlException("Handler takes one argument: a message") - + def wrapper(master, message): if not hasattr(message, "reply"): raise exceptions.ControlException("Message %s has no reply attribute" % message) @@ -172,7 +189,8 @@ def handler(f): handling = True message.reply.handled = True - ret = f(*args, **kwargs) + with master.handlecontext(): + ret = f(master, message) if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() @@ -216,7 +234,7 @@ class Reply(object): def __del__(self): if not self.acked: # This will be ignored by the interpreter, but emit a warning - raise exceptions.ControlException("Un-acked message") + raise exceptions.ControlException("Un-acked message: %s" % self.obj) class DummyReply(object): diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 520f82e9..7590a3fa 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -89,9 +89,10 @@ class FlowMaster(controller.Master): Raises: ScriptException """ - s = script.Script(command, script.ScriptContext(self)) + s = script.Script(command) s.load() if use_reloader: + s.reply = controller.DummyReply() script.reloader.watch(s, lambda: self.event_queue.put(("script_change", s))) self.scripts.append(s) @@ -234,8 +235,12 @@ class FlowMaster(controller.Master): return super(FlowMaster, self).tick(timeout) def duplicate_flow(self, f): + """ + Duplicate flow, and insert it into state without triggering any of + the normal flow events. + """ f2 = f.copy() - self.load_flow(f2) + self.state.add_flow(f2) return f2 def create_request(self, method, scheme, host, port, path): @@ -479,14 +484,14 @@ class FlowMaster(controller.Master): s.unload() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}":\n{}'.format(s.filename, e), 'error') + self.add_event('Error reloading "{}":\n{}'.format(s.path, e), 'error') try: s.load() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}":\n{}'.format(s.filename, e), 'error') + self.add_event('Error reloading "{}":\n{}'.format(s.path, e), 'error') else: - self.add_event('"{}" reloaded.'.format(s.filename), 'info') + self.add_event('"{}" reloaded.'.format(s.path), 'info') return ok @controller.handler diff --git a/mitmproxy/script/__init__.py b/mitmproxy/script/__init__.py index d6bff4c7..9a3985ab 100644 --- a/mitmproxy/script/__init__.py +++ b/mitmproxy/script/__init__.py @@ -1,12 +1,10 @@ from . import reloader from .concurrent import concurrent from .script import Script -from .script_context import ScriptContext from ..exceptions import ScriptException __all__ = [ "Script", - "ScriptContext", "concurrent", "ScriptException", "reloader" diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index 56d39d0b..010a5fa0 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -18,9 +18,9 @@ def concurrent(fn): "Concurrent decorator not supported for '%s' method." % fn.__name__ ) - def _concurrent(ctx, obj): + def _concurrent(obj): def run(): - fn(ctx, obj) + fn(obj) if not obj.reply.acked: obj.reply.ack() obj.reply.take() diff --git a/mitmproxy/script/reloader.py b/mitmproxy/script/reloader.py index 50401034..857d76cd 100644 --- a/mitmproxy/script/reloader.py +++ b/mitmproxy/script/reloader.py @@ -15,8 +15,8 @@ _observers = {} def watch(script, callback): if script in _observers: raise RuntimeError("Script already observed") - script_dir = os.path.dirname(os.path.abspath(script.filename)) - script_name = os.path.basename(script.filename) + script_dir = os.path.dirname(os.path.abspath(script.path)) + script_name = os.path.basename(script.path) event_handler = _ScriptModificationHandler(callback, filename=script_name) observer = Observer() observer.schedule(event_handler, script_dir) diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py index 9ff79f52..db4909ca 100644 --- a/mitmproxy/script/script.py +++ b/mitmproxy/script/script.py @@ -6,38 +6,40 @@ by the mitmproxy-specific ScriptContext. # Do not import __future__ here, this would apply transitively to the inline scripts. from __future__ import absolute_import, print_function, division -import inspect import os import shlex import sys import contextlib -import warnings import six +from typing import List # noqa from mitmproxy import exceptions @contextlib.contextmanager -def setargs(args): +def scriptenv(path, args): + # type: (str, List[str]) -> None oldargs = sys.argv - sys.argv = args + script_dir = os.path.dirname(os.path.abspath(path)) + + sys.argv = [path] + args + sys.path.append(script_dir) try: yield finally: sys.argv = oldargs + sys.path.pop() class Script(object): - """ Script object representing an inline script. """ - def __init__(self, command, context): + def __init__(self, command): self.command = command - self.args = self.parse_command(command) - self.ctx = context + self.path, self.args = self.parse_command(command) self.ns = None def __enter__(self): @@ -46,15 +48,15 @@ class Script(object): def __exit__(self, exc_type, exc_val, exc_tb): if exc_val: - return False # reraise the exception + return False # re-raise the exception self.unload() - @property - def filename(self): - return self.args[0] - @staticmethod def parse_command(command): + # type: (str) -> Tuple[str,List[str]] + """ + Returns a (path, args) tuple. + """ if not command or not command.strip(): raise exceptions.ScriptException("Empty script command.") # Windows: escape all backslashes in the path. @@ -71,7 +73,7 @@ class Script(object): args[0]) elif os.path.isdir(args[0]): raise exceptions.ScriptException("Not a file: %s" % args[0]) - return args + return args[0], args[1:] def load(self): """ @@ -85,31 +87,19 @@ class Script(object): """ if self.ns is not None: raise exceptions.ScriptException("Script is already loaded") - script_dir = os.path.dirname(os.path.abspath(self.args[0])) - self.ns = {'__file__': os.path.abspath(self.args[0])} - sys.path.append(script_dir) - sys.path.append(os.path.join(script_dir, "..")) - try: - with open(self.filename) as f: - code = compile(f.read(), self.filename, 'exec') - exec(code, self.ns, self.ns) - except Exception: - six.reraise( - exceptions.ScriptException, - exceptions.ScriptException.from_exception_context(), - sys.exc_info()[2] - ) - finally: - sys.path.pop() - sys.path.pop() - - start_fn = self.ns.get("start") - if start_fn and len(inspect.getargspec(start_fn).args) == 2: - warnings.warn( - "The 'args' argument of the start() script hook is deprecated. " - "Please use sys.argv instead." - ) - return self.run("start", self.args) + self.ns = {'__file__': os.path.abspath(self.path)} + + with scriptenv(self.path, self.args): + try: + with open(self.path) as f: + code = compile(f.read(), self.path, 'exec') + exec(code, self.ns, self.ns) + except Exception: + six.reraise( + exceptions.ScriptException, + exceptions.ScriptException.from_exception_context(), + sys.exc_info()[2] + ) return self.run("start") def unload(self): @@ -134,8 +124,8 @@ class Script(object): f = self.ns.get(name) if f: try: - with setargs(self.args): - return f(self.ctx, *args, **kwargs) + with scriptenv(self.path, self.args): + return f(*args, **kwargs) except Exception: six.reraise( exceptions.ScriptException, diff --git a/mitmproxy/script/script_context.py b/mitmproxy/script/script_context.py deleted file mode 100644 index 44e2736b..00000000 --- a/mitmproxy/script/script_context.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -The mitmproxy script context provides an API to inline scripts. -""" -from __future__ import absolute_import, print_function, division - -from mitmproxy import contentviews - - -class ScriptContext(object): - - """ - The script context should be used to interact with the global mitmproxy state from within a - script. - """ - - def __init__(self, master): - self._master = master - - def log(self, message, level="info"): - """ - Logs an event. - - By default, only events with level "error" get displayed. This can be controlled with the "-v" switch. - How log messages are handled depends on the front-end. mitmdump will print them to stdout, - mitmproxy sends output to the eventlog for display ("e" keyboard shortcut). - """ - self._master.add_event(message, level) - - def kill_flow(self, f): - """ - Kills a flow immediately. No further data will be sent to the client or the server. - """ - f.kill(self._master) - - def duplicate_flow(self, f): - """ - Returns a duplicate of the specified flow. The flow is also - injected into the current state, and is ready for editing, replay, - etc. - """ - self._master.pause_scripts = True - f = self._master.duplicate_flow(f) - self._master.pause_scripts = False - return f - - def replay_request(self, f): - """ - Replay the request on the current flow. The response will be added - to the flow object. - """ - return self._master.replay_request(f, block=True, run_scripthooks=False) - - @property - def app_registry(self): - return self._master.apps - - def add_contentview(self, view_obj): - contentviews.add(view_obj) - - def remove_contentview(self, view_obj): - contentviews.remove(view_obj) diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 62541f3f..57eeca19 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -11,7 +11,7 @@ class Thing: @tutils.skip_appveyor def test_concurrent(): - with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py"), None) as s: + with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py")) as s: f1, f2 = Thing(), Thing() s.run("request", f1) s.run("request", f2) @@ -23,6 +23,6 @@ def test_concurrent(): def test_concurrent_err(): - s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py"), None) + s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py")) with tutils.raises("Concurrent decorator not supported for 'start' method"): s.load() diff --git a/test/mitmproxy/script/test_reloader.py b/test/mitmproxy/script/test_reloader.py index 0345f6ed..e33903b9 100644 --- a/test/mitmproxy/script/test_reloader.py +++ b/test/mitmproxy/script/test_reloader.py @@ -10,7 +10,7 @@ def test_simple(): pass script = mock.Mock() - script.filename = "foo.py" + script.path = "foo.py" e = Event() diff --git a/test/mitmproxy/script/test_script.py b/test/mitmproxy/script/test_script.py index fe98fab5..48fe65c9 100644 --- a/test/mitmproxy/script/test_script.py +++ b/test/mitmproxy/script/test_script.py @@ -21,21 +21,21 @@ class TestParseCommand: def test_parse_args(self): with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data/scripts/a.py") == ["data/scripts/a.py"] - assert Script.parse_command("data/scripts/a.py foo bar") == ["data/scripts/a.py", "foo", "bar"] - assert Script.parse_command("data/scripts/a.py 'foo bar'") == ["data/scripts/a.py", "foo bar"] + assert Script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", []) + assert Script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"]) + assert Script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"]) @tutils.skip_not_windows def test_parse_windows(self): with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data\\scripts\\a.py") == ["data\\scripts\\a.py"] - assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ["data\\scripts\\a.py", 'foo \\ bar'] + assert Script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", []) + assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar']) def test_simple(): with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("a.py --var 42", None) - assert s.filename == "a.py" + s = Script("a.py --var 42") + assert s.path == "a.py" assert s.ns is None s.load() @@ -50,34 +50,34 @@ def test_simple(): with tutils.raises(ScriptException): s.run("here") - with Script("a.py --var 42", None) as s: + with Script("a.py --var 42") as s: s.run("here") def test_script_exception(): with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("syntaxerr.py", None) + s = Script("syntaxerr.py") with tutils.raises(ScriptException): s.load() - s = Script("starterr.py", None) + s = Script("starterr.py") with tutils.raises(ScriptException): s.load() - s = Script("a.py", None) + s = Script("a.py") s.load() with tutils.raises(ScriptException): s.load() - s = Script("a.py", None) + s = Script("a.py") with tutils.raises(ScriptException): s.run("here") with tutils.raises(ScriptException): - with Script("reqerr.py", None) as s: + with Script("reqerr.py") as s: s.run("request", None) - s = Script("unloaderr.py", None) + s = Script("unloaderr.py") s.load() with tutils.raises(ScriptException): s.unload() diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index f30973e7..3b5ff2a2 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,47 +1,31 @@ import glob import json +import mock import os import sys from contextlib import contextmanager from mitmproxy import script -from mitmproxy.proxy import config import netlib.utils from netlib import tutils as netutils from netlib.http import Headers -from . import tservers, tutils +from . import tutils example_dir = netlib.utils.Data(__name__).path("../../examples") -class DummyContext(object): - """Emulate script.ScriptContext() functionality.""" - - contentview = None - - def log(self, *args, **kwargs): - pass - - def add_contentview(self, view_obj): - self.contentview = view_obj - - def remove_contentview(self, view_obj): - self.contentview = None - - @contextmanager def example(command): command = os.path.join(example_dir, command) - ctx = DummyContext() - with script.Script(command, ctx) as s: + with script.Script(command) as s: yield s -def test_load_scripts(): +@mock.patch("mitmproxy.master") +@mock.patch("mitmproxy.log") +def test_load_scripts(log, master): scripts = glob.glob("%s/*.py" % example_dir) - tmaster = tservers.TestMaster(config.ProxyConfig()) - for f in scripts: if "har_extractor" in f: continue @@ -54,7 +38,7 @@ def test_load_scripts(): if "modify_response_body" in f: f += " foo bar" # two arguments required - s = script.Script(f, script.ScriptContext(tmaster)) + s = script.Script(f) try: s.load() except Exception as v: @@ -71,17 +55,21 @@ def test_add_header(): assert flow.response.headers["newheader"] == "foo" -def test_custom_contentviews(): - with example("custom_contentviews.py") as ex: - pig = ex.ctx.contentview +@mock.patch("mitmproxy.contentviews.remove") +@mock.patch("mitmproxy.contentviews.add") +def test_custom_contentviews(add, remove): + with example("custom_contentviews.py"): + assert add.called + pig = add.call_args[0][0] _, fmt = pig(b"test!") assert any(b'esttay!' in val[0][1] for val in fmt) assert not pig(b"gobbledygook") + assert remove.called def test_iframe_injector(): with tutils.raises(script.ScriptException): - with example("iframe_injector.py") as ex: + with example("iframe_injector.py"): pass flow = tutils.tflow(resp=netutils.tresp(content=b"mitmproxy")) @@ -121,7 +109,7 @@ def test_modify_response_body(): flow = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) with example("modify_response_body.py mitmproxy rocks") as ex: - assert ex.ctx.old == b"mitmproxy" and ex.ctx.new == b"rocks" + assert ex.ns["state"]["old"] == b"mitmproxy" and ex.ns["state"]["new"] == b"rocks" ex.run("response", flow) assert flow.response.content == b"I <3 rocks" @@ -133,7 +121,8 @@ def test_redirect_requests(): assert flow.request.host == "mitmproxy.org" -def test_har_extractor(): +@mock.patch("mitmproxy.log") +def test_har_extractor(log): if sys.version_info >= (3, 0): with tutils.raises("does not work on Python 3"): with example("har_extractor.py -"): @@ -159,4 +148,4 @@ def test_har_extractor(): with open(tutils.test_data.path("data/har_extractor.har")) as fp: test_data = json.load(fp) - assert json.loads(ex.ctx.HARLog.json()) == test_data["test_response"] + assert json.loads(ex.ns["context"].HARLog.json()) == test_data["test_response"] diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 0ab7624e..9dd8b79c 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -1,6 +1,7 @@ import os import socket import time +import types from OpenSSL import SSL from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.tcp import Address @@ -945,7 +946,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): f.reply.kill() return _func(f) - setattr(master, attr, handler) + setattr(master, attr, types.MethodType(handler, master)) kill_requests( self.chain[1].tmaster, -- cgit v1.2.3 From 6da166d762815229af825b18f32fb0a8f9b6c710 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 8 Jul 2016 10:37:56 +0200 Subject: http2: fix self-depended streams --- mitmproxy/protocol/http2.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 83ff71e4..4b57174b 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -206,37 +206,46 @@ class Http2Layer(base.Layer): self.streams[event.pushed_stream_id].request_data_finished.set() self.streams[event.pushed_stream_id].start() elif isinstance(event, events.PriorityUpdated): + mapped_stream_id = event.stream_id + if mapped_stream_id in self.streams and self.streams[mapped_stream_id].server_stream_id: + # if the stream is already up and running and was sent to the server + # use the mapped server stream id to update priority information + mapped_stream_id = self.streams[mapped_stream_id].server_stream_id + if eid in self.streams: if self.streams[eid].handled_priority_event is event: - # This event was already handled during stream creation + # this event was already handled during stream creation # HeadersFrame + Priority information as RequestReceived return True - if eid in self.streams: - self.streams[eid].priority_weight = event.weight - self.streams[eid].priority_depends_on = event.depends_on - self.streams[eid].priority_exclusive = event.exclusive - - stream_id = event.stream_id - if stream_id in self.streams.keys() and self.streams[stream_id].server_stream_id: - stream_id = self.streams[stream_id].server_stream_id - - depends_on = event.depends_on - if depends_on in self.streams.keys() and self.streams[depends_on].server_stream_id: - depends_on = self.streams[depends_on].server_stream_id + self.streams[eid].priority_weight = event.weight + self.streams[eid].priority_depends_on = event.depends_on + self.streams[eid].priority_exclusive = event.exclusive with self.server_conn.h2.lock: self.server_conn.h2.prioritize( - stream_id, + mapped_stream_id, weight=event.weight, - depends_on=depends_on, + depends_on=self._map_depends_on_stream_id(mapped_stream_id, event.depends_on), exclusive=event.exclusive ) self.server_conn.send(self.server_conn.h2.data_to_send()) elif isinstance(event, events.TrailersReceived): - raise NotImplementedError() + raise NotImplementedError("TrailersReceived not implemented") return True + def _map_depends_on_stream_id(self, stream_id, depends_on): + mapped_depends_on = depends_on + if mapped_depends_on in self.streams and self.streams[mapped_depends_on].server_stream_id: + # if the depends-on-stream is already up and running and was sent to the server + # use the mapped server stream id to update priority information + mapped_depends_on = self.streams[mapped_depends_on].server_stream_id + if stream_id == mapped_depends_on: + # looks like one of the streams wasn't opened yet + # prevent self-dependent streams which result in ProtocolError + mapped_depends_on += 2 + return mapped_depends_on + def _cleanup_streams(self): death_time = time.time() - 10 for stream_id in self.streams.keys(): @@ -412,8 +421,6 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) headers.insert(0, ":path", message.path) headers.insert(0, ":method", message.method) headers.insert(0, ":scheme", message.scheme) - self.server_stream_id = self.server_conn.h2.get_next_available_stream_id() - self.server_to_client_stream_ids[self.server_stream_id] = self.client_stream_id try: self.server_conn.h2.safe_send_headers( @@ -422,7 +429,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) headers, end_stream=self.no_body, priority_weight=self.priority_weight, - priority_depends_on=self.priority_depends_on, + priority_depends_on=self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on), priority_exclusive=self.priority_exclusive, ) except Exception as e: -- cgit v1.2.3 From df43a962d7a49be7d34d4af1b1e9d50349e4ea36 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 8 Jul 2016 10:54:21 +0200 Subject: simplify tox test exclusion --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6f189d35..29414486 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py test/mitmproxy/test_flow_format_compat.py test/mitmproxy/test_examples.py test/mitmproxy/test_protocol_http2.py test/mitmproxy/test_flow.py + TESTS = --ignore=test/mitmproxy/console/test_master.py --ignore=test/mitmproxy/test_dump.py test/ HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From c9482c62554154fb46d3e1a205f0d04593e52abd Mon Sep 17 00:00:00 2001 From: jpkrause Date: Fri, 8 Jul 2016 12:46:28 -0700 Subject: Added flow converters for versions 0.11 and 0.12 --- mitmproxy/flow/io_compat.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index ec825f71..3dc0cbce 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -7,6 +7,13 @@ import six from netlib import version, strutils +def convert_011_012(data): + data[b"version"] = (0, 12) + return data + +def convert_012_013(data): + data[b"version"] = (0, 13) + return data def convert_013_014(data): data[b"request"][b"first_line_format"] = data[b"request"].pop(b"form_in") @@ -97,6 +104,8 @@ def convert_unicode(data): converters = { + (0, 11): convert_011_012, + (0, 12): convert_012_013, (0, 13): convert_013_014, (0, 14): convert_014_015, (0, 15): convert_015_016, -- cgit v1.2.3 From 05ab9c1b376473265af1a779c59319cc99b5fcd3 Mon Sep 17 00:00:00 2001 From: jpkrause Date: Fri, 8 Jul 2016 15:03:36 -0700 Subject: Updated coding style and modified flow format compat test to use v0.11 and v0.10 dump files --- mitmproxy/flow/io_compat.py | 3 +++ test/mitmproxy/data/dumpfile-010 | Bin 0 -> 2140 bytes test/mitmproxy/data/dumpfile-011 | Bin 0 -> 5465 bytes test/mitmproxy/data/dumpfile-012 | 35 ------------------------------ test/mitmproxy/data/dumpfile-013 | 35 ------------------------------ test/mitmproxy/test_flow_format_compat.py | 4 ++-- 6 files changed, 5 insertions(+), 72 deletions(-) create mode 100644 test/mitmproxy/data/dumpfile-010 create mode 100644 test/mitmproxy/data/dumpfile-011 delete mode 100644 test/mitmproxy/data/dumpfile-012 delete mode 100644 test/mitmproxy/data/dumpfile-013 diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index 3dc0cbce..bcfbd375 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -7,14 +7,17 @@ import six from netlib import version, strutils + def convert_011_012(data): data[b"version"] = (0, 12) return data + def convert_012_013(data): data[b"version"] = (0, 13) return data + def convert_013_014(data): data[b"request"][b"first_line_format"] = data[b"request"].pop(b"form_in") data[b"request"][b"http_version"] = b"HTTP/" + ".".join( diff --git a/test/mitmproxy/data/dumpfile-010 b/test/mitmproxy/data/dumpfile-010 new file mode 100644 index 00000000..435795bf Binary files /dev/null and b/test/mitmproxy/data/dumpfile-010 differ diff --git a/test/mitmproxy/data/dumpfile-011 b/test/mitmproxy/data/dumpfile-011 new file mode 100644 index 00000000..2534ad89 Binary files /dev/null and b/test/mitmproxy/data/dumpfile-011 differ diff --git a/test/mitmproxy/data/dumpfile-012 b/test/mitmproxy/data/dumpfile-012 deleted file mode 100644 index 49c2350d..00000000 --- a/test/mitmproxy/data/dumpfile-012 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:12#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}} \ No newline at end of file diff --git a/test/mitmproxy/data/dumpfile-013 b/test/mitmproxy/data/dumpfile-013 deleted file mode 100644 index ede06f23..00000000 --- a/test/mitmproxy/data/dumpfile-013 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:13#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}} \ No newline at end of file diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py index b2cef88d..cc80db81 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_flow_format_compat.py @@ -4,7 +4,7 @@ from . import tutils def test_load(): - with open(tutils.test_data.path("data/dumpfile-013"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f: flow_reader = FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -12,7 +12,7 @@ def test_load(): def test_cannot_convert(): - with open(tutils.test_data.path("data/dumpfile-012"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f: flow_reader = FlowReader(f) with tutils.raises(FlowReadException): list(flow_reader.stream()) -- cgit v1.2.3 From 0a1431ed2c638ceb34308d38250eab2717640c49 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 8 Jul 2016 15:49:38 -0700 Subject: fix #1314 --- netlib/debug.py | 43 +++++++++++++++++++++++-------------------- setup.py | 1 - 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/netlib/debug.py b/netlib/debug.py index a395afcb..fcd72a21 100644 --- a/netlib/debug.py +++ b/netlib/debug.py @@ -7,8 +7,6 @@ import signal import platform import traceback -import psutil - from netlib import version from OpenSSL import SSL @@ -40,15 +38,32 @@ def sysinfo(): def dump_info(sig, frm, file=sys.stdout): # pragma: no cover - p = psutil.Process() - print("****************************************************", file=file) print("Summary", file=file) print("=======", file=file) - print("num threads: ", p.num_threads(), file=file) - if hasattr(p, "num_fds"): - print("num fds: ", p.num_fds(), file=file) - print("memory: ", p.memory_info(), file=file) + + try: + import psutil + except: + print("(psutil not installed, skipping some debug info)", file=file) + else: + p = psutil.Process() + print("num threads: ", p.num_threads(), file=file) + if hasattr(p, "num_fds"): + print("num fds: ", p.num_fds(), file=file) + print("memory: ", p.memory_info(), file=file) + + print(file=file) + print("Files", file=file) + print("=====", file=file) + for i in p.open_files(): + print(i, file=file) + + print(file=file) + print("Connections", file=file) + print("===========", file=file) + for i in p.connections(): + print(i, file=file) print(file=file) print("Threads", file=file) @@ -63,18 +78,6 @@ def dump_info(sig, frm, file=sys.stdout): # pragma: no cover for i in bthreads: print(i._threadinfo(), file=file) - print(file=file) - print("Files", file=file) - print("=====", file=file) - for i in p.open_files(): - print(i, file=file) - - print(file=file) - print("Connections", file=file) - print("===========", file=file) - for i in p.connections(): - print(i, file=file) - print("****************************************************", file=file) diff --git a/setup.py b/setup.py index 0de4ba32..3e5171f9 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( "lxml>=3.5.0, <3.7", "Pillow>=3.2, <3.3", "passlib>=1.6.5, <1.7", - "psutil>=4.2, <4.4", "pyasn1>=0.1.9, <0.2", "pyOpenSSL>=16.0, <17.0", "pyparsing>=2.1.3, <2.2", -- cgit v1.2.3 From f7639e077ab10dc8e7f180f44b531b9ad35c060b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 8 Jul 2016 15:58:12 -0700 Subject: py3++ --- test/mitmproxy/console/test_master.py | 4 +++- test/mitmproxy/test_dump.py | 2 ++ tox.ini | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index 33261c28..d42863b3 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -112,11 +112,13 @@ def test_options(): class TestMaster(mastertest.MasterTest): def mkmaster(self, filt, **options): + if "verbosity" not in options: + options["verbosity"] = 0 o = console.master.Options(filtstr=filt, **options) return console.master.ConsoleMaster(None, o) def test_basic(self): m = self.mkmaster(None) for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 234490f8..18de09d5 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -63,6 +63,8 @@ class TestDumpMaster(mastertest.MasterTest): def mkmaster(self, filt, **options): cs = StringIO() + if "verbosity" not in options: + options["verbosity"] = 0 o = dump.Options(filtstr=filt, **options) return dump.DumpMaster(None, o, outfile=cs) diff --git a/tox.ini b/tox.ini index 29414486..e33fa4e0 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = --ignore=test/mitmproxy/console/test_master.py --ignore=test/mitmproxy/test_dump.py test/ + TESTS = --ignore=test/mitmproxy/test_dump.py test/ HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From 5d2b7c52f9c33e84be5c4330b09b0f2a5ad869e2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 8 Jul 2016 19:57:57 -0700 Subject: move script context to mitmproxy.ctx --- examples/har_extractor.py | 6 +++--- examples/nonblocking.py | 4 ++-- examples/proxapp.py | 4 ++-- examples/stub.py | 20 ++++++++++---------- examples/tls_passthrough.py | 2 +- mitmproxy/__init__.py | 5 ----- mitmproxy/controller.py | 12 ++++++------ mitmproxy/ctx.py | 4 ++++ test/mitmproxy/data/scripts/all.py | 14 +++++++------- test/mitmproxy/data/scripts/duplicate_flow.py | 4 ++-- test/mitmproxy/test_examples.py | 6 +++--- 11 files changed, 40 insertions(+), 41 deletions(-) create mode 100644 mitmproxy/ctx.py diff --git a/examples/har_extractor.py b/examples/har_extractor.py index 208a2fa8..2a69b9af 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -219,17 +219,17 @@ def done(): compressed_json_dump = context.HARLog.compress() if context.dump_file == '-': - mitmproxy.log(pprint.pformat(json.loads(json_dump))) + mitmproxy.ctx.log(pprint.pformat(json.loads(json_dump))) elif context.dump_file.endswith('.zhar'): file(context.dump_file, "w").write(compressed_json_dump) else: file(context.dump_file, "w").write(json_dump) - mitmproxy.log( + mitmproxy.ctx.log( "HAR log finished with %s bytes (%s bytes compressed)" % ( len(json_dump), len(compressed_json_dump) ) ) - mitmproxy.log( + mitmproxy.ctx.log( "Compression rate is %s%%" % str( 100. * len(compressed_json_dump) / len(json_dump) ) diff --git a/examples/nonblocking.py b/examples/nonblocking.py index 05a26921..b81478df 100644 --- a/examples/nonblocking.py +++ b/examples/nonblocking.py @@ -5,6 +5,6 @@ from mitmproxy.script import concurrent @concurrent # Remove this and see what happens def request(flow): - mitmproxy.log("handle request: %s%s" % (flow.request.host, flow.request.path)) + mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path)) time.sleep(5) - mitmproxy.log("start request: %s%s" % (flow.request.host, flow.request.path)) + mitmproxy.ctx.log("start request: %s%s" % (flow.request.host, flow.request.path)) diff --git a/examples/proxapp.py b/examples/proxapp.py index ddc38544..2935b587 100644 --- a/examples/proxapp.py +++ b/examples/proxapp.py @@ -17,9 +17,9 @@ def hello_world(): # Register the app using the magic domain "proxapp" on port 80. Requests to # this domain and port combination will now be routed to the WSGI app instance. def start(): - mitmproxy.master.apps.add(app, "proxapp", 80) + mitmproxy.ctx.master.apps.add(app, "proxapp", 80) # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set) # but won't send any data. - mitmproxy.master.apps.add(app, "example.com", 443) + mitmproxy.ctx.master.apps.add(app, "example.com", 443) diff --git a/examples/stub.py b/examples/stub.py index a4f16699..10b34283 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -8,7 +8,7 @@ def start(): """ Called once on script startup, before any other events. """ - mitmproxy.log("start") + mitmproxy.ctx.log("start") def clientconnect(root_layer): @@ -16,14 +16,14 @@ def clientconnect(root_layer): Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests """ - mitmproxy.log("clientconnect") + mitmproxy.ctx.log("clientconnect") def request(flow): """ Called when a client request has been received. """ - mitmproxy.log("request") + mitmproxy.ctx.log("request") def serverconnect(server_conn): @@ -31,7 +31,7 @@ def serverconnect(server_conn): Called when the proxy initiates a connection to the target server. Note that a connection can correspond to multiple HTTP requests """ - mitmproxy.log("serverconnect") + mitmproxy.ctx.log("serverconnect") def responseheaders(flow): @@ -40,14 +40,14 @@ def responseheaders(flow): but the response body has not been processed yet. Can be used to tell mitmproxy to stream the response. """ - mitmproxy.log("responseheaders") + mitmproxy.ctx.log("responseheaders") def response(flow): """ Called when a server response has been received. """ - mitmproxy.log("response") + mitmproxy.ctx.log("response") def error(flow): @@ -56,25 +56,25 @@ def error(flow): interrupted connections. This is distinct from a valid server HTTP error response, which is simply a response with an HTTP error code. """ - mitmproxy.log("error") + mitmproxy.ctx.log("error") def serverdisconnect(server_conn): """ Called when the proxy closes the connection to the target server. """ - mitmproxy.log("serverdisconnect") + mitmproxy.ctx.log("serverdisconnect") def clientdisconnect(root_layer): """ Called when a client disconnects from the proxy. """ - mitmproxy.log("clientdisconnect") + mitmproxy.ctx.log("clientdisconnect") def done(): """ Called once on script shutdown, after any other events. """ - mitmproxy.log("done") + mitmproxy.ctx.log("done") diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py index 374020e7..20e8f9be 100644 --- a/examples/tls_passthrough.py +++ b/examples/tls_passthrough.py @@ -135,7 +135,7 @@ def next_layer(next_layer): next_layer.__class__ = TlsFeedback else: # We don't intercept - reply with a pass-through layer and add a "skipped" entry. - mitmproxy.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") + mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) next_layer.reply.send(next_layer_replacement) tls_strategy.record_skipped(server_address) diff --git a/mitmproxy/__init__.py b/mitmproxy/__init__.py index 99ce7c9c..e69de29b 100644 --- a/mitmproxy/__init__.py +++ b/mitmproxy/__init__.py @@ -1,5 +0,0 @@ -from typing import Callable # noqa -from mitmproxy import flow # noqa - -master = None # type: flow.FlowMaster -log = None # type: Callable[[str], None] diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 222ebc69..e2be3a53 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -6,7 +6,7 @@ import contextlib from six.moves import queue -import mitmproxy +from . import ctx as mitmproxy_ctx from netlib import basethread from . import exceptions @@ -59,16 +59,16 @@ class Master(object): @contextlib.contextmanager def handlecontext(self): # Handlecontexts also have to nest - leave cleanup to the outermost - if mitmproxy.master: + if mitmproxy_ctx.master: yield return - mitmproxy.master = self - mitmproxy.log = Log(self) + mitmproxy_ctx.master = self + mitmproxy_ctx.log = Log(self) try: yield finally: - mitmproxy.master = None - mitmproxy.log = None + mitmproxy_ctx.master = None + mitmproxy_ctx.log = None def add_server(self, server): # We give a Channel to the server which can be used to communicate with the master diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py new file mode 100644 index 00000000..fcfdfd0b --- /dev/null +++ b/mitmproxy/ctx.py @@ -0,0 +1,4 @@ +from typing import Callable # noqa + +master = None # type: "mitmproxy.flow.FlowMaster" +log = None # type: Callable[[str], None] diff --git a/test/mitmproxy/data/scripts/all.py b/test/mitmproxy/data/scripts/all.py index 17ffe33f..bf8e93ec 100644 --- a/test/mitmproxy/data/scripts/all.py +++ b/test/mitmproxy/data/scripts/all.py @@ -3,35 +3,35 @@ log = [] def clientconnect(cc): - mitmproxy.log("XCLIENTCONNECT") + mitmproxy.ctx.log("XCLIENTCONNECT") log.append("clientconnect") def serverconnect(cc): - mitmproxy.log("XSERVERCONNECT") + mitmproxy.ctx.log("XSERVERCONNECT") log.append("serverconnect") def request(f): - mitmproxy.log("XREQUEST") + mitmproxy.ctx.log("XREQUEST") log.append("request") def response(f): - mitmproxy.log("XRESPONSE") + mitmproxy.ctx.log("XRESPONSE") log.append("response") def responseheaders(f): - mitmproxy.log("XRESPONSEHEADERS") + mitmproxy.ctx.log("XRESPONSEHEADERS") log.append("responseheaders") def clientdisconnect(cc): - mitmproxy.log("XCLIENTDISCONNECT") + mitmproxy.ctx.log("XCLIENTDISCONNECT") log.append("clientdisconnect") def error(cc): - mitmproxy.log("XERROR") + mitmproxy.ctx.log("XERROR") log.append("error") diff --git a/test/mitmproxy/data/scripts/duplicate_flow.py b/test/mitmproxy/data/scripts/duplicate_flow.py index a50d2cb5..565b1845 100644 --- a/test/mitmproxy/data/scripts/duplicate_flow.py +++ b/test/mitmproxy/data/scripts/duplicate_flow.py @@ -2,5 +2,5 @@ import mitmproxy def request(f): - f = mitmproxy.master.duplicate_flow(f) - mitmproxy.master.replay_request(f, block=True, run_scripthooks=False) + f = mitmproxy.ctx.master.duplicate_flow(f) + mitmproxy.ctx.master.replay_request(f, block=True, run_scripthooks=False) diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 3b5ff2a2..bdadcd11 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -21,8 +21,8 @@ def example(command): yield s -@mock.patch("mitmproxy.master") -@mock.patch("mitmproxy.log") +@mock.patch("mitmproxy.ctx.master") +@mock.patch("mitmproxy.ctx.log") def test_load_scripts(log, master): scripts = glob.glob("%s/*.py" % example_dir) @@ -121,7 +121,7 @@ def test_redirect_requests(): assert flow.request.host == "mitmproxy.org" -@mock.patch("mitmproxy.log") +@mock.patch("mitmproxy.ctx.log") def test_har_extractor(log): if sys.version_info >= (3, 0): with tutils.raises("does not work on Python 3"): -- cgit v1.2.3 From b4469d25794ce05448bbfbe82088eb22b0de84e3 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 28 Jun 2016 20:54:02 +0530 Subject: py3++: test_dump --- mitmproxy/dump.py | 6 +++--- mitmproxy/models/http.py | 3 ++- tox.ini | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 6670be9b..3625d86e 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -248,7 +248,7 @@ class DumpMaster(flow.FlowMaster): else: client = click.style("[replay]", fg="yellow", bold=True) - method = flow.request.method + method = flow.request.data.method method_color = dict( GET="green", DELETE="red" @@ -258,7 +258,7 @@ class DumpMaster(flow.FlowMaster): url = flow.request.pretty_url else: url = flow.request.url - url = click.style(strutils.bytes_to_escaped_str(url), bold=True) + url = click.style(url, bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): @@ -288,7 +288,7 @@ class DumpMaster(flow.FlowMaster): elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(strutils.bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True) + reason = click.style(strutils.bytes_to_escaped_str(flow.response.data.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 8aeb84d6..df546b9b 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, print_function, division import cgi +import six from mitmproxy.models.flow import Flow from netlib import encoding @@ -220,7 +221,7 @@ class HTTPFlow(Flow): If f is a string, it will be compiled as a filter expression. If the expression is invalid, ValueError is raised. """ - if isinstance(f, str): + if isinstance(f, six.string_types): from .. import filt f = filt.parse(f) diff --git a/tox.ini b/tox.ini index e33fa4e0..411718c8 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = --ignore=test/mitmproxy/test_dump.py test/ + TESTS = test/ HOME = {envtmpdir} [testenv:docs] -- cgit v1.2.3 From 147f61fa62f3c2b29f46c6a033bea19c4c49a171 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 7 Jul 2016 19:37:33 +0530 Subject: py3++ --- mitmproxy/contentviews.py | 2 +- mitmproxy/dump.py | 10 +++++----- test/mitmproxy/test_dump.py | 41 +++++++++++++++++++++++------------------ 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 331a706f..706fa5ed 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -172,7 +172,7 @@ class ViewHex(View): def _format(data): for offset, hexa, s in strutils.hexdump(data): yield [ - ("offset", offset + " "), + ("offset", offset + b" "), ("text", hexa + " "), ("text", s) ] diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 3625d86e..10465b63 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -244,21 +244,21 @@ class DumpMaster(flow.FlowMaster): stickycookie = "" if flow.client_conn: - client = click.style(strutils.bytes_to_escaped_str(flow.client_conn.address.host), bold=True) + client = click.style(strutils.escape_control_characters(flow.client_conn.address.host), bold=True) else: client = click.style("[replay]", fg="yellow", bold=True) - method = flow.request.data.method + method = flow.request.method method_color = dict( GET="green", DELETE="red" ).get(method.upper(), "magenta") - method = click.style(strutils.bytes_to_escaped_str(method), fg=method_color, bold=True) + method = click.style(strutils.escape_control_characters(method), fg=method_color, bold=True) if self.showhost: url = flow.request.pretty_url else: url = flow.request.url - url = click.style(url, bold=True) + url = click.style(strutils.escape_control_characters(url), bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): @@ -288,7 +288,7 @@ class DumpMaster(flow.FlowMaster): elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(strutils.bytes_to_escaped_str(flow.response.data.reason), fg=code_color, bold=True) + reason = click.style(strutils.escape_control_characters(flow.response.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 18de09d5..263c8b3d 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -40,7 +40,7 @@ def test_strfuncs(): flow.response.status_code = 300 m.echo_flow(flow) - flow = tutils.tflow(resp=netlib.tutils.tresp(content="{")) + flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 m.echo_flow(flow) @@ -70,15 +70,20 @@ class TestDumpMaster(mastertest.MasterTest): def test_basic(self): for i in (1, 2, 3): - assert "GET" in self.dummy_cycle(self.mkmaster("~s", flow_detail=i), 1, "") assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), 1, - "\x00\x00\x00" + b"" ) assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), - 1, "ascii" + 1, + b"\x00\x00\x00" + ) + assert "GET" in self.dummy_cycle( + self.mkmaster("~s", flow_detail=i), + 1, + b"ascii" ) def test_error(self): @@ -115,12 +120,12 @@ class TestDumpMaster(mastertest.MasterTest): o = dump.Options(server_replay=[p], kill=True) m = dump.DumpMaster(None, o, outfile=cs) - self.cycle(m, "content") - self.cycle(m, "content") + self.cycle(m, b"content") + self.cycle(m, b"content") o = dump.Options(server_replay=[p], kill=False) m = dump.DumpMaster(None, o, outfile=cs) - self.cycle(m, "nonexistent") + self.cycle(m, b"nonexistent") o = dump.Options(client_replay=[p], kill=False) m = dump.DumpMaster(None, o, outfile=cs) @@ -131,7 +136,7 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) assert "GET" in self.dummy_cycle( self.mkmaster(None, flow_detail=1, rfile=p), - 0, "", + 0, b"", ) tutils.raises( @@ -149,7 +154,7 @@ class TestDumpMaster(mastertest.MasterTest): def test_filter(self): assert "GET" not in self.dummy_cycle( - self.mkmaster("~u foo", verbosity=1), 1, "" + self.mkmaster("~u foo", verbosity=1), 1, b"" ) def test_app(self): @@ -162,21 +167,21 @@ class TestDumpMaster(mastertest.MasterTest): cs = StringIO() o = dump.Options(replacements=[(".*", "content", "foo")]) m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") - assert f.request.content == "foo" + f = self.cycle(m, b"content") + assert f.request.content == b"foo" def test_setheader(self): cs = StringIO() o = dump.Options(setheaders=[(".*", "one", "two")]) m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") + f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" def test_write(self): with tutils.tmpdir() as d: p = os.path.join(d, "a") self.dummy_cycle( - self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, "" + self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1 @@ -185,11 +190,11 @@ class TestDumpMaster(mastertest.MasterTest): p = os.path.join(d, "a.append") self.dummy_cycle( self.mkmaster(None, outfile=(p, "wb"), verbosity=0), - 1, "" + 1, b"" ) self.dummy_cycle( self.mkmaster(None, outfile=(p, "ab"), verbosity=0), - 1, "" + 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2 @@ -205,7 +210,7 @@ class TestDumpMaster(mastertest.MasterTest): None, scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1 ), - 1, "", + 1, b"", ) assert "XCLIENTCONNECT" in ret assert "XSERVERCONNECT" in ret @@ -226,11 +231,11 @@ class TestDumpMaster(mastertest.MasterTest): def test_stickycookie(self): self.dummy_cycle( self.mkmaster(None, stickycookie = ".*"), - 1, "" + 1, b"" ) def test_stickyauth(self): self.dummy_cycle( self.mkmaster(None, stickyauth = ".*"), - 1, "" + 1, b"" ) -- cgit v1.2.3 From c7cbbf5f0de9bceccc37b73586ed3baf9421f63a Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 9 Jul 2016 12:32:41 +0530 Subject: Set verbosity & flow_detail to 0 --- test/mitmproxy/test_dump.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 263c8b3d..aa73b5a4 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -65,6 +65,8 @@ class TestDumpMaster(mastertest.MasterTest): cs = StringIO() if "verbosity" not in options: options["verbosity"] = 0 + if "flow_detail" not in options: + options["flow_detail"] = 0 o = dump.Options(filtstr=filt, **options) return dump.DumpMaster(None, o, outfile=cs) @@ -118,16 +120,22 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) o = dump.Options(server_replay=[p], kill=True) + o.verbosity = 0 + o.flow_detail = 0 m = dump.DumpMaster(None, o, outfile=cs) self.cycle(m, b"content") self.cycle(m, b"content") o = dump.Options(server_replay=[p], kill=False) + o.verbosity = 0 + o.flow_detail = 0 m = dump.DumpMaster(None, o, outfile=cs) self.cycle(m, b"nonexistent") o = dump.Options(client_replay=[p], kill=False) + o.verbosity = 0 + o.flow_detail = 0 m = dump.DumpMaster(None, o, outfile=cs) def test_read(self): @@ -166,6 +174,8 @@ class TestDumpMaster(mastertest.MasterTest): def test_replacements(self): cs = StringIO() o = dump.Options(replacements=[(".*", "content", "foo")]) + o.verbosity = 0 + o.flow_detail = 0 m = dump.DumpMaster(None, o, outfile=cs) f = self.cycle(m, b"content") assert f.request.content == b"foo" @@ -173,6 +183,8 @@ class TestDumpMaster(mastertest.MasterTest): def test_setheader(self): cs = StringIO() o = dump.Options(setheaders=[(".*", "one", "two")]) + o.verbosity = 0 + o.flow_detail = 0 m = dump.DumpMaster(None, o, outfile=cs) f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" -- cgit v1.2.3 From 0a21d270046f97a8d7da4378ca3a54b4a7f898b8 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 9 Jul 2016 12:43:21 +0530 Subject: strutils.hexdump returns native_strings --- mitmproxy/contentviews.py | 2 +- netlib/strutils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 706fa5ed..331a706f 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -172,7 +172,7 @@ class ViewHex(View): def _format(data): for offset, hexa, s in strutils.hexdump(data): yield [ - ("offset", offset + b" "), + ("offset", offset + " "), ("text", hexa + " "), ("text", s) ] diff --git a/netlib/strutils.py b/netlib/strutils.py index 9208f954..a51df886 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -146,7 +146,7 @@ def hexdump(s): A generator of (offset, hex, str) tuples """ for i in range(0, len(s), 16): - offset = "{:0=10x}".format(i).encode() + offset = "{:0=10x}".format(i) part = s[i:i + 16] x = " ".join("{:0=2x}".format(i) for i in six.iterbytes(part)) x = x.ljust(47) # 16*2 + 15 -- cgit v1.2.3 From bd58230178efac789fabc4e4aa81ae6ad5c4b2b4 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 9 Jul 2016 12:51:43 +0530 Subject: Remove py35 section from tox.ini --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index 411718c8..1896744d 100644 --- a/tox.ini +++ b/tox.ini @@ -14,11 +14,6 @@ commands = py.test --timeout 60 {posargs} {env:TESTS} {env:CI_COMMANDS:python -c ""} -[testenv:py35] -setenv = - TESTS = test/ - HOME = {envtmpdir} - [testenv:docs] changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html -- cgit v1.2.3 From 83a1cc5a9a62dbe22bd9e87f496928ae1664da2b Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 9 Jul 2016 12:57:55 +0530 Subject: Make escape_control_characters handle strings on Py2 --- netlib/strutils.py | 4 ++-- test/netlib/test_strutils.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/netlib/strutils.py b/netlib/strutils.py index a51df886..32e77927 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -57,8 +57,8 @@ def escape_control_characters(text, keep_spacing=True): Args: keep_spacing: If True, tabs and newlines will not be replaced. """ - # type: (six.text_type) -> six.text_type - if not isinstance(text, six.text_type): + # type: (six.string_types) -> six.text_type + if not isinstance(text, six.string_types): raise ValueError("text type must be unicode but is {}".format(type(text).__name__)) trans = _control_char_trans_newline if keep_spacing else _control_char_trans diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 68bfdb94..7c3eacc6 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -38,8 +38,9 @@ def test_escape_control_characters(): u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' ) - with tutils.raises(ValueError): - strutils.escape_control_characters(b"foo") + if not six.PY2: + with tutils.raises(ValueError): + strutils.escape_control_characters(b"foo") def test_bytes_to_escaped_str(): -- cgit v1.2.3 From 3d40fae6d759adc4d0862c52cb043cd3c9fa0cc8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 9 Jul 2016 01:05:04 -0700 Subject: travis: move osx test to the top OSX is the slow one, so it should start straight away. --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0e3c370..eda57384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,12 @@ matrix: include: - python: 3.5 env: TOXENV=lint + - language: generic + os: osx + osx_image: xcode7.1 + git: + depth: 9999999 + env: TOXENV=py27 - python: 3.5 env: TOXENV=py35 - python: 3.5 @@ -28,12 +34,6 @@ matrix: env: TOXENV=py27 - python: 2.7 env: TOXENV=py27 NO_ALPN=1 - - language: generic - os: osx - osx_image: xcode7.1 - git: - depth: 9999999 - env: TOXENV=py27 - python: 3.5 env: TOXENV=docs allow_failures: -- cgit v1.2.3 From e16a632b95924e43bc6e708fbb2d4703a41c8951 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 9 Jul 2016 01:25:37 -0700 Subject: remove pathod doc leftovers --- .sources/bootswatch.less | 171 -------------------------------------- .sources/make | 5 -- .sources/variables.less | 208 ----------------------------------------------- 3 files changed, 384 deletions(-) delete mode 100644 .sources/bootswatch.less delete mode 100644 .sources/make delete mode 100644 .sources/variables.less diff --git a/.sources/bootswatch.less b/.sources/bootswatch.less deleted file mode 100644 index f9e4b827..00000000 --- a/.sources/bootswatch.less +++ /dev/null @@ -1,171 +0,0 @@ -// Bootswatch.less -// Swatch: Journal -// Version: 2.0.4 -// ----------------------------------------------------- - -// TYPOGRAPHY -// ----------------------------------------------------- - -@import url('https://fonts.googleapis.com/css?family=Open+Sans:400,700'); - -h1, h2, h3, h4, h5, h6, .navbar .brand { - font-weight: 700; -} - -// SCAFFOLDING -// ----------------------------------------------------- - -a { - text-decoration: none; -} - -.nav a, .navbar .brand, .subnav a, a.btn, .dropdown-menu a { - text-decoration: none; -} - -// NAVBAR -// ----------------------------------------------------- - -.navbar { - - .navbar-inner { - @shadow: 0 2px 4px rgba(0,0,0,.25), inset 0 -1px 0 rgba(0,0,0,.1); - .box-shadow(@shadow); - border-top: 1px solid #E5E5E5; - .border-radius(0); - } - - .brand { - text-shadow: none; - - &:hover { - background-color: #EEEEEE; - } - } - - .navbar-text { - line-height: 68px; - } - - .nav > li > a { - text-shadow: none; - } - - .dropdown-menu { - .border-radius(0); - } - - .nav li.dropdown.active > .dropdown-toggle, - .nav li.dropdown.active > .dropdown-toggle:hover, - .nav li.dropdown.open > .dropdown-toggle, - .nav li.dropdown.active.open > .dropdown-toggle, - .nav li.dropdown.active.open > .dropdown-toggle:hover { - background-color: @grayLighter; - color: @linkColor; - } - - .nav li.dropdown .dropdown-toggle .caret, - .nav .open .caret, - .nav .open .dropdown-toggle:hover .caret { - border-top-color: @black; - opacity: 1; - } - - .nav-collapse.in .nav li > a:hover { - background-color: @grayLighter; - } - - .nav-collapse .nav li > a { - color: @textColor; - text-decoration: none; - font-weight: normal; - } - - .nav-collapse .navbar-form, - .nav-collapse .navbar-search { - border-color: transparent; - } - - .navbar-search .search-query, - .navbar-search .search-query:hover { - border: 1px solid @grayLighter; - color: @textColor; - .placeholder(@gray); - } -} - -div.subnav { - background-color: @bodyBackground; - background-image: none; - @shadow: 0 1px 2px rgba(0,0,0,.25); - .box-shadow(@shadow); - .border-radius(0); - - &.subnav-fixed { - top: @navbarHeight; - } - - .nav > li > a:hover, - .nav > .active > a, - .nav > .active > a:hover { - color: @textColor; - text-decoration: none; - font-weight: normal; - } - - .nav > li:first-child > a, - .nav > li:first-child > a:hover { - .border-radius(0); - } -} - -// BUTTONS -// ----------------------------------------------------- - -.btn-primary { - .buttonBackground(lighten(@linkColor, 5%), @linkColor); -} - -[class^="icon-"], [class*=" icon-"] { - vertical-align: -2px; -} - -// MODALS -// ----------------------------------------------------- - -.modal { - .border-radius(0px); - background: @bodyBackground; -} - -.modal-header { - border-bottom: none; -} - -.modal-header .close { - text-decoration: none; -} - -.modal-footer { - background: transparent; - .box-shadow(none); - border-top: none; -} - - -// MISC -// ----------------------------------------------------- - -code, pre, pre.prettyprint, .well { - background-color: @grayLighter; -} - -.hero-unit { - .box-shadow(inset 0 1px 1px rgba(0,0,0,.05)); - border: 1px solid rgba(0,0,0,.05); - .border-radius(0); -} - -.table-bordered, .well, .prettyprint { - .border-radius(0); -} diff --git a/.sources/make b/.sources/make deleted file mode 100644 index 94648859..00000000 --- a/.sources/make +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -pygmentize -f html ../examples/test_context.py > ../pathod/templates/examples_context.html -pygmentize -f html ../examples/test_setup.py > ../pathod/templates/examples_setup.html -pygmentize -f html ../examples/test_setupall.py > ../pathod/templates/examples_setupall.html -pygmentize -f html ../examples/pathod_pathoc.py > ../pathod/templates/pathod_pathoc.html diff --git a/.sources/variables.less b/.sources/variables.less deleted file mode 100644 index 75ff5be6..00000000 --- a/.sources/variables.less +++ /dev/null @@ -1,208 +0,0 @@ -// Variables.less -// Variables to customize the look and feel of Bootstrap -// Swatch: Journal -// Version: 2.0.4 -// ----------------------------------------------------- - -// GLOBAL VALUES -// -------------------------------------------------- - - -// Grays -// ------------------------- -@black: #000; -@grayDarker: #222; -@grayDark: #333; -@gray: #888; -@grayLight: #999; -@grayLighter: #eee; -@white: #fff; - - -// Accent colors -// ------------------------- -@blue: #4380D3; -@blueDark: darken(@blue, 15%); -@green: #22B24C; -@red: #C00; -@yellow: #FCFADB; -@orange: #FF7F00; -@pink: #CC99CC; -@purple: #7a43b6; -@tan: #FFCA73; - - - -// Scaffolding -// ------------------------- -@bodyBackground: #FCFBFD; -@textColor: @grayDarker; - - -// Links -// ------------------------- -@linkColor: @blue; -@linkColorHover: @red; - - -// Typography -// ------------------------- -@sansFontFamily: 'Open Sans', "Helvetica Neue", Helvetica, Arial, sans-serif; -@serifFontFamily: Georgia, "Times New Roman", Times, serif; -@monoFontFamily: Menlo, Monaco, Consolas, "Courier New", monospace; - -@baseFontSize: 14px; -@baseFontFamily: @sansFontFamily; -@baseLineHeight: 18px; -@altFontFamily: @serifFontFamily; - -@headingsFontFamily: inherit; // empty to use BS default, @baseFontFamily -@headingsFontWeight: bold; // instead of browser default, bold -@headingsColor: inherit; // empty to use BS default, @textColor - - -// Tables -// ------------------------- -@tableBackground: transparent; // overall background-color -@tableBackgroundAccent: @grayLighter; // for striping -@tableBackgroundHover: #f5f5f5; // for hover -@tableBorder: #ddd; // table and cell border - - -// Buttons -// ------------------------- -@btnBackground: @white; -@btnBackgroundHighlight: darken(@white, 10%); -@btnBorder: darken(@white, 20%); - -@btnPrimaryBackground: @linkColor; -@btnPrimaryBackgroundHighlight: spin(@btnPrimaryBackground, 15%); - -@btnInfoBackground: #5bc0de; -@btnInfoBackgroundHighlight: #2f96b4; - -@btnSuccessBackground: #62c462; -@btnSuccessBackgroundHighlight: #51a351; - -@btnWarningBackground: lighten(@orange, 10%); -@btnWarningBackgroundHighlight: @orange; - -@btnDangerBackground: #ee5f5b; -@btnDangerBackgroundHighlight: #bd362f; - -@btnInverseBackground: @linkColor; -@btnInverseBackgroundHighlight: darken(@linkColor, 5%); - - -// Forms -// ------------------------- -@inputBackground: @white; -@inputBorder: #ccc; -@inputBorderRadius: 3px; -@inputDisabledBackground: @grayLighter; -@formActionsBackground: @grayLighter; - -// Dropdowns -// ------------------------- -@dropdownBackground: @bodyBackground; -@dropdownBorder: rgba(0,0,0,.2); -@dropdownLinkColor: @textColor; -@dropdownLinkColorHover: @textColor; -@dropdownLinkBackgroundHover: #eee; -@dropdownDividerTop: #e5e5e5; -@dropdownDividerBottom: @white; - - - -// COMPONENT VARIABLES -// -------------------------------------------------- - -// Z-index master list -// ------------------------- -// Used for a bird's eye view of components dependent on the z-axis -// Try to avoid customizing these :) -@zindexDropdown: 1000; -@zindexPopover: 1010; -@zindexTooltip: 1020; -@zindexFixedNavbar: 1030; -@zindexModalBackdrop: 1040; -@zindexModal: 1050; - - -// Sprite icons path -// ------------------------- -@iconSpritePath: "../img/glyphicons-halflings.png"; -@iconWhiteSpritePath: "../img/glyphicons-halflings-white.png"; - - -// Input placeholder text color -// ------------------------- -@placeholderText: @grayLight; - - -// Hr border color -// ------------------------- -@hrBorder: @grayLighter; - - -// Navbar -// ------------------------- -@navbarHeight: 50px; -@navbarBackground: @bodyBackground; -@navbarBackgroundHighlight: @bodyBackground; - -@navbarText: @textColor; -@navbarLinkColor: @linkColor; -@navbarLinkColorHover: @linkColor; -@navbarLinkColorActive: @navbarLinkColorHover; -@navbarLinkBackgroundHover: @grayLighter; -@navbarLinkBackgroundActive: @grayLighter; - -@navbarSearchBackground: lighten(@navbarBackground, 25%); -@navbarSearchBackgroundFocus: @white; -@navbarSearchBorder: darken(@navbarSearchBackground, 30%); -@navbarSearchPlaceholderColor: #ccc; -@navbarBrandColor: @blue; - - -// Hero unit -// ------------------------- -@heroUnitBackground: @grayLighter; -@heroUnitHeadingColor: inherit; -@heroUnitLeadColor: inherit; - - -// Form states and alerts -// ------------------------- -@warningText: #c09853; -@warningBackground: #fcf8e3; -@warningBorder: darken(spin(@warningBackground, -10), 3%); - -@errorText: #b94a48; -@errorBackground: #f2dede; -@errorBorder: darken(spin(@errorBackground, -10), 3%); - -@successText: #468847; -@successBackground: #dff0d8; -@successBorder: darken(spin(@successBackground, -10), 5%); - -@infoText: #3a87ad; -@infoBackground: #d9edf7; -@infoBorder: darken(spin(@infoBackground, -10), 7%); - - - -// GRID -// -------------------------------------------------- - -// Default 940px grid -// ------------------------- -@gridColumns: 12; -@gridColumnWidth: 60px; -@gridGutterWidth: 20px; -@gridRowWidth: (@gridColumns * @gridColumnWidth) + (@gridGutterWidth * (@gridColumns - 1)); - -// Fluid grid -// ------------------------- -@fluidGridColumnWidth: 6.382978723%; -@fluidGridGutterWidth: 2.127659574%; -- cgit v1.2.3 From 6f0a733d62277611403a9c70ee9188d5a6cc881e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 9 Jul 2016 01:29:12 -0700 Subject: Add Python 3.5 markers to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 3e5171f9..d1c15734 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,8 @@ setup( "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Security", -- cgit v1.2.3 From 636cbfd1631670b71f9b156c73fc6345a87def48 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 9 Jul 2016 01:33:56 -0700 Subject: Update README.rst --- README.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 52584719..3ae2eaab 100644 --- a/README.rst +++ b/README.rst @@ -52,9 +52,9 @@ If you want to contribute changes, keep on reading. Hacking ------- -To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with +To get started hacking on mitmproxy, make sure you have Python_ 3.5.x or above with virtualenv_ installed (you can find installation instructions for virtualenv -here_). Then do the following: +`here `_). Then do the following: .. code-block:: text @@ -194,7 +194,6 @@ PR checks will fail and block merging. We are using this command to check for st .. _Python: https://www.python.org/ .. _virtualenv: http://virtualenv.readthedocs.org/en/latest/ -.. _here: http://virtualenv.readthedocs.org/en/latest/installation.html .. _autoenv: https://github.com/kennethreitz/autoenv .. _.env: https://github.com/mitmproxy/mitmproxy/blob/master/.env .. _Sphinx: http://sphinx-doc.org/ -- cgit v1.2.3 From 718fa5bbcd12c64bfc4bd19c0c27e7d82da21a84 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 9 Jul 2016 01:39:37 -0700 Subject: remove unused files --- .dockerignore | 1 - .python-version | 2 -- Dockerfile | 4 ---- 3 files changed, 7 deletions(-) delete mode 100644 .dockerignore delete mode 100644 .python-version delete mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 6b8710a7..00000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.git diff --git a/.python-version b/.python-version deleted file mode 100644 index 2339c8bf..00000000 --- a/.python-version +++ /dev/null @@ -1,2 +0,0 @@ -2.7.11 -3.5.1 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a689ed5e..00000000 --- a/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM mitmproxy/base:latest-onbuild -EXPOSE 8080 -EXPOSE 8081 -VOLUME /certs -- cgit v1.2.3 From aa1615eafd0b538ac289d901c3096b5e9c8bcbc2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 11:19:58 +0200 Subject: cleanup tox --- tox.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 1896744d..9da23a2e 100644 --- a/tox.ini +++ b/tox.ini @@ -7,16 +7,14 @@ deps = {env:CI_DEPS:} -rrequirements.txt passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* -setenv = - TESTS = test/ - HOME = {envtmpdir} +setenv = HOME = {envtmpdir} commands = - py.test --timeout 60 {posargs} {env:TESTS} + py.test --timeout 60 {posargs} {env:CI_COMMANDS:python -c ""} [testenv:docs] changedir = docs -commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] deps = flake8>=2.6.2, <3 -- cgit v1.2.3 From 8cfa5ad3062c9c84c1606845f3ba08922c3d8320 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 11:29:25 +0200 Subject: update osx travis --- .travis.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index eda57384..21e99f1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,9 @@ matrix: include: - python: 3.5 env: TOXENV=lint - - language: generic - os: osx - osx_image: xcode7.1 - git: - depth: 9999999 - env: TOXENV=py27 + - os: osx + language: generic + env: TOXENV=py35 - python: 3.5 env: TOXENV=py35 - python: 3.5 @@ -45,7 +42,7 @@ install: then brew update || brew update # try again if it fails brew outdated openssl || brew upgrade openssl - brew install python + brew install python python3 fi - pip install tox @@ -55,9 +52,9 @@ after_success: - | if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then - pip install -U virtualenv - ./dev.sh - source venv/bin/activate + pip3 install -U virtualenv + ./dev.sh 3.5 + source venv3.5/bin/activate pip install -e ./release python ./release/rtool.py bdist python ./release/rtool.py upload-snapshot --bdist --wheel -- cgit v1.2.3 From 3fe5b47cf9f5292aaba13077bf70937d5237bf26 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 11:44:18 +0200 Subject: try to fix weird install behaviour --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 092f834a..ead8d1c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' +https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; (sys_platform == 'win32' and python_version == '3.5') -e .[dev,examples,contentviews] --e ./release \ No newline at end of file +-e ./release -- cgit v1.2.3 From 3a3d9f65e693ff0e0c6eb35c872d7a8b43f06dce Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 11:50:30 +0200 Subject: upgrade pip on osx travis --- .travis.yml | 2 ++ requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 21e99f1f..3aa2e75c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,8 @@ install: brew update || brew update # try again if it fails brew outdated openssl || brew upgrade openssl brew install python python3 + pip install -U pip setuptools wheel + pip3 install -U pip3 setuptools wheel fi - pip install tox diff --git a/requirements.txt b/requirements.txt index ead8d1c2..f6fdfda9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; (sys_platform == 'win32' and python_version == '3.5') +https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' -e .[dev,examples,contentviews] -e ./release -- cgit v1.2.3 From 3e2d41d507d9622991f10f5e75788098063f2134 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 11:55:52 +0200 Subject: fix pip install on osx travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3aa2e75c..65200ac1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,7 @@ install: brew outdated openssl || brew upgrade openssl brew install python python3 pip install -U pip setuptools wheel - pip3 install -U pip3 setuptools wheel + pip3 install -U pip setuptools wheel fi - pip install tox -- cgit v1.2.3 From 7efd63f94c5871ab6f93a1e39355cb37d9d6e107 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 12:17:43 +0200 Subject: add whitespace to improve url parsing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f6fdfda9..ce37652d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' +https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl ; sys_platform == 'win32' and python_version == '3.5' -e .[dev,examples,contentviews] -e ./release -- cgit v1.2.3 From 608435cabf03e759118f2314490dcee5539f6f66 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 7 Jul 2016 23:25:39 +0530 Subject: Delete stickycookies when told by the server Fixes #1096 --- mitmproxy/flow/modules.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index cba96fbc..46da5b64 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,8 +1,10 @@ from __future__ import absolute_import, print_function, division import collections +import email.utils import hashlib import re +import time from six.moves import http_cookiejar from six.moves import urllib @@ -320,10 +322,33 @@ class StickyCookieState: for name, (value, attrs) in f.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with # valid RFC 822/1123 datetime specifications for expiry. Sigh. - a = self.ckey(attrs, f) - if self.domain_match(f.request.host, a[0]): - b = attrs.with_insert(0, name, value) - self.jar[a][name] = b + dom_port_path = self.ckey(attrs, f) + + if self.domain_match(f.request.host, dom_port_path[0]): + + # See if 'expires' time is in the past + expired = False + if 'expires' in attrs: + e = email.utils.parsedate_tz(attrs["expires"]) + if e: + exp_ts = email.utils.mktime_tz(e) + now_ts = time.time() + expired = exp_ts < now_ts + + # or if Max-Age is 0 + expired = expired or (int(attrs.get('Max-Age', 1)) == 0) + + if expired: + # Remove the cookie from jar + self.jar[dom_port_path].pop(name, None) + + # If all cookies of a dom_port_path have been removed + # then remove it from the jar itself + if not self.jar[dom_port_path]: + self.jar.pop(dom_port_path, None) + else: + b = attrs.with_insert(0, name, value) + self.jar[dom_port_path][name] = b def handle_request(self, f): l = [] -- cgit v1.2.3 From 6a746deff57d7283ee8440148b87ea16a672739a Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 7 Jul 2016 23:26:52 +0530 Subject: Add tests for deletion of stickycookies --- test/mitmproxy/test_flow.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index bf7622f6..c0d9155f 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -55,14 +55,16 @@ class TestStickyCookieState: assert s.domain_match("google.com", ".google.com") def test_response(self): - c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ + c = ( + "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " + ) s, f = self._response(c, "host") assert not s.jar.keys() s, f = self._response(c, "www.google.com") - assert s.jar.keys() + assert list(s.jar.keys())[0] == ('.google.com', 80, '/') s, f = self._response("SSID=mooo", "www.google.com") assert list(s.jar.keys())[0] == ('www.google.com', 80, '/') @@ -101,6 +103,28 @@ class TestStickyCookieState: assert len(s.jar[googlekey]) == 1 assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue" + def test_response_delete(self): + c = "duffer=zafar; Path=/", "www.google.com" + + # Test that a cookie is be deleted + # by setting the expire time in the past + s, f = self._response(*c) + f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT" + s.handle_response(f) + assert not s.jar.keys() + + # or by setting Max-Age to 0 + s, f = self._response(*c) + f.response.headers["Set-Cookie"] = "duffer=; Max-Age=0" + s.handle_response(f) + assert not s.jar.keys() + + # or both + s, f = self._response(*c) + f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0" + s.handle_response(f) + assert not s.jar.keys() + def test_request(self): s, f = self._response("SSID=mooo", b"www.google.com") assert "cookie" not in f.request.headers -- cgit v1.2.3 From c92992f03bba6553ec39fc42e6716beb942967e3 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 8 Jul 2016 14:16:29 +0530 Subject: Move cookie expiry detection to separate function --- mitmproxy/flow/modules.py | 17 +---------------- netlib/http/cookies.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index 46da5b64..ab41da8d 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,10 +1,8 @@ from __future__ import absolute_import, print_function, division import collections -import email.utils import hashlib import re -import time from six.moves import http_cookiejar from six.moves import urllib @@ -325,20 +323,7 @@ class StickyCookieState: dom_port_path = self.ckey(attrs, f) if self.domain_match(f.request.host, dom_port_path[0]): - - # See if 'expires' time is in the past - expired = False - if 'expires' in attrs: - e = email.utils.parsedate_tz(attrs["expires"]) - if e: - exp_ts = email.utils.mktime_tz(e) - now_ts = time.time() - expired = exp_ts < now_ts - - # or if Max-Age is 0 - expired = expired or (int(attrs.get('Max-Age', 1)) == 0) - - if expired: + if cookies.is_expired(attrs): # Remove the cookie from jar self.jar[dom_port_path].pop(name, None) diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index 768a85df..90789365 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -1,7 +1,8 @@ import collections +import email.utils import re +import time -import email.utils from netlib import multidict """ @@ -260,3 +261,24 @@ def refresh_set_cookie_header(c, delta): if not ret: raise ValueError("Invalid Cookie") return ret + +def is_expired(cookie_attrs): + """ + Determines whether a cookie has expired. + + Returns: boolean + """ + expired = False + + # See if 'expires' time is in the past + if 'expires' in cookie_attrs: + e = email.utils.parsedate_tz(cookie_attrs["expires"]) + if e: + exp_ts = email.utils.mktime_tz(e) + now_ts = time.time() + expired = exp_ts < now_ts + + # or if Max-Age is 0 + expired = expired or (int(cookie_attrs.get('Max-Age', 1)) == 0) + + return expired -- cgit v1.2.3 From 39f51084003b93a2e9868f7a56acfc29c12ed79e Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 10 Jul 2016 01:06:50 +0530 Subject: Test cookies.is_expired separately --- netlib/http/cookies.py | 13 +++++++++---- test/netlib/http/test_cookies.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index 90789365..dd0af99c 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -262,23 +262,28 @@ def refresh_set_cookie_header(c, delta): raise ValueError("Invalid Cookie") return ret + def is_expired(cookie_attrs): """ Determines whether a cookie has expired. Returns: boolean """ - expired = False # See if 'expires' time is in the past + expires = False if 'expires' in cookie_attrs: e = email.utils.parsedate_tz(cookie_attrs["expires"]) if e: exp_ts = email.utils.mktime_tz(e) now_ts = time.time() - expired = exp_ts < now_ts + expires = exp_ts < now_ts # or if Max-Age is 0 - expired = expired or (int(cookie_attrs.get('Max-Age', 1)) == 0) + max_age = False + try: + max_age = int(cookie_attrs.get('Max-Age', 1)) == 0 + except ValueError: + pass - return expired + return expires or max_age diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py index 83b85656..17e21b94 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/netlib/http/test_cookies.py @@ -245,3 +245,24 @@ def test_refresh_cookie(): assert cookies.refresh_set_cookie_header(c, 0) c = "foo/bar=bla" assert cookies.refresh_set_cookie_header(c, 0) + + +def test_is_expired(): + CA = cookies.CookieAttrs + + # A cookie can be expired + # by setting the expire time in the past + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) + + # or by setting Max-Age to 0 + assert cookies.is_expired(CA([("Max-Age", "0")])) + + # or both + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")])) + + assert not cookies.is_expired(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")])) + assert not cookies.is_expired(CA([("Max-Age", "1")])) + assert not cookies.is_expired(CA([("Expires", "Thu, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")])) + + assert not cookies.is_expired(CA([("Max-Age", "nan")])) + assert not cookies.is_expired(CA([("Expires", "false")])) -- cgit v1.2.3 From 7eade1ef7c24b98567c1657973290aa5377b2719 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 10 Jul 2016 01:08:02 +0530 Subject: Remove redundant tests --- test/mitmproxy/test_flow.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index c0d9155f..74b3f599 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -113,18 +113,6 @@ class TestStickyCookieState: s.handle_response(f) assert not s.jar.keys() - # or by setting Max-Age to 0 - s, f = self._response(*c) - f.response.headers["Set-Cookie"] = "duffer=; Max-Age=0" - s.handle_response(f) - assert not s.jar.keys() - - # or both - s, f = self._response(*c) - f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT; Max-Age=0" - s.handle_response(f) - assert not s.jar.keys() - def test_request(self): s, f = self._response("SSID=mooo", b"www.google.com") assert "cookie" not in f.request.headers -- cgit v1.2.3 From 22c0db3b19cf14b0056d4fc32c62f31095c2e382 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 22:31:00 +0200 Subject: revert travis osx changes --- .travis.yml | 19 ++++++++++--------- requirements.txt | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65200ac1..eda57384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,9 +20,12 @@ matrix: include: - python: 3.5 env: TOXENV=lint - - os: osx - language: generic - env: TOXENV=py35 + - language: generic + os: osx + osx_image: xcode7.1 + git: + depth: 9999999 + env: TOXENV=py27 - python: 3.5 env: TOXENV=py35 - python: 3.5 @@ -42,9 +45,7 @@ install: then brew update || brew update # try again if it fails brew outdated openssl || brew upgrade openssl - brew install python python3 - pip install -U pip setuptools wheel - pip3 install -U pip setuptools wheel + brew install python fi - pip install tox @@ -54,9 +55,9 @@ after_success: - | if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then - pip3 install -U virtualenv - ./dev.sh 3.5 - source venv3.5/bin/activate + pip install -U virtualenv + ./dev.sh + source venv/bin/activate pip install -e ./release python ./release/rtool.py bdist python ./release/rtool.py upload-snapshot --bdist --wheel diff --git a/requirements.txt b/requirements.txt index ce37652d..092f834a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl ; sys_platform == 'win32' and python_version == '3.5' +https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' -e .[dev,examples,contentviews] --e ./release +-e ./release \ No newline at end of file -- cgit v1.2.3 From 8135b356d665245e7b45de53c36f79c8d6a42565 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 10 Jul 2016 15:50:13 +0530 Subject: Decode commit value in rtool --- release/rtool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/rtool.py b/release/rtool.py index 04e1249d..4e43eaef 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -76,7 +76,7 @@ def get_snapshot_version(): return "{version}dev{tag_dist:04}-0x{commit}".format( version=get_version(), # this should already be the next version tag_dist=tag_dist, - commit=commit + commit=commit.decode() ) -- cgit v1.2.3 From ff63fadf0b34df1739a549a17998e56fb0d18f9b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 9 Jul 2016 14:39:12 +0200 Subject: move OSX Travis job to py3 for snapshots --- .travis.yml | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index eda57384..e9566ebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,10 @@ matrix: include: - python: 3.5 env: TOXENV=lint - - language: generic - os: osx - osx_image: xcode7.1 - git: - depth: 9999999 - env: TOXENV=py27 + - os: osx + osx_image: xcode7.3 + language: generic + env: TOXENV=py35 - python: 3.5 env: TOXENV=py35 - python: 3.5 @@ -44,20 +42,26 @@ install: if [[ $TRAVIS_OS_NAME == "osx" ]] then brew update || brew update # try again if it fails - brew outdated openssl || brew upgrade openssl - brew install python + brew upgrade + brew reinstall openssl + brew reinstall pyenv + eval "$(pyenv init -)" + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.2 + pyenv global 3.5.2 + pyenv shell 3.5.2 + pip install -U pip setuptools wheel virtualenv fi - pip install tox -script: set -o pipefail; tox -- --cov netlib --cov mitmproxy --cov pathod 2>&1 | grep -v Cryptography_locking_cb +script: set -o pipefail; python -m tox -- --cov netlib --cov mitmproxy --cov pathod 2>&1 | grep -v Cryptography_locking_cb after_success: - | if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then - pip install -U virtualenv - ./dev.sh - source venv/bin/activate + git fetch --unshallow + ./dev.sh 3.5 + source venv3.5/bin/activate pip install -e ./release python ./release/rtool.py bdist python ./release/rtool.py upload-snapshot --bdist --wheel @@ -71,4 +75,7 @@ notifications: on_failure: change on_start: never -cache: pip +cache: + directories: + - $HOME/.pyenv + - $HOME/.cache/pip -- cgit v1.2.3 From c90de8b9a40124c9e859d00d995e3e8133941a12 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 10 Jul 2016 13:16:23 +0200 Subject: fix sysinfo for py3 --- netlib/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netlib/debug.py b/netlib/debug.py index fcd72a21..29c7f655 100644 --- a/netlib/debug.py +++ b/netlib/debug.py @@ -17,7 +17,7 @@ def sysinfo(): "Mitmproxy version: %s" % version.VERSION, "Python version: %s" % platform.python_version(), "Platform: %s" % platform.platform(), - "SSL version: %s" % SSL.SSLeay_version(SSL.SSLEAY_VERSION), + "SSL version: %s" % SSL.SSLeay_version(SSL.SSLEAY_VERSION).decode(), ] d = platform.linux_distribution() t = "Linux distro: %s %s %s" % d -- cgit v1.2.3 From e89defe9ad4d1660b061d9fdf92b5c325faa2000 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 10 Jul 2016 13:38:49 +0200 Subject: update dependency and loosen restrictions on good citizens --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index d1c15734..e0bd4545 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,13 @@ from setuptools import setup, find_packages from codecs import open import os +from netlib import version + # Based on https://github.com/pypa/sampleproject/blob/master/setup.py # and https://python-packaging-user-guide.readthedocs.org/ here = os.path.abspath(os.path.dirname(__file__)) -from netlib import version - with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() @@ -72,7 +72,7 @@ setup( "html2text>=2016.1.8, <=2016.5.29", "hyperframe>=4.0.1, <5", "lxml>=3.5.0, <3.7", - "Pillow>=3.2, <3.3", + "Pillow>=3.2, <3.4", "passlib>=1.6.5, <1.7", "pyasn1>=0.1.9, <0.2", "pyOpenSSL>=16.0, <17.0", @@ -100,10 +100,10 @@ setup( 'dev': [ "tox>=2.3, <3", "mock>=2.0, <2.1", - "pytest>=2.8.7, <2.10", - "pytest-cov>=2.2.1, <2.3", - "pytest-timeout>=1.0.0, <1.1", - "pytest-xdist>=1.14, <1.15", + "pytest>=2.8.7, <3", + "pytest-cov>=2.2.1, <3", + "pytest-timeout>=1.0.0, <2", + "pytest-xdist>=1.14, <2", "sphinx>=1.3.5, <1.5", "sphinx-autobuild>=0.5.2, <0.7", "sphinxcontrib-documentedlist>=0.4.0, <0.5", -- cgit v1.2.3 From b611997619b126efdc371e725cbb742a6b02c410 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 10 Jul 2016 18:53:00 +0200 Subject: http2: simplify test class --- test/mitmproxy/test_protocol_http2.py | 87 +++++++++-------------------------- 1 file changed, 22 insertions(+), 65 deletions(-) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 58ffb787..a4f6b574 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -151,8 +151,7 @@ class _Http2TestBase(object): wfile.flush() -@requires_alpn -class TestSimple(_Http2TestBase, _Http2ServerBase): +class _Http2Test(_Http2TestBase, _Http2ServerBase): @classmethod def setup_class(self): @@ -164,6 +163,10 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): _Http2TestBase.teardown_class() _Http2ServerBase.teardown_class() + +@requires_alpn +class TestSimple(_Http2Test): + @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): @@ -196,14 +199,18 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): def test_simple(self): client, h2_conn = self._setup_connection() - self._send_request(client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:%s" % self.server.server.address.port), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('ClIeNt-FoO', 'client-bar-1'), - ('ClIeNt-FoO', 'client-bar-2'), - ], body=b'my request body echoed back to me') + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('ClIeNt-FoO', 'client-bar-1'), + ('ClIeNt-FoO', 'client-bar-2'), + ], + body=b'my request body echoed back to me') done = False while not done: @@ -233,19 +240,9 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestWithBodies(_Http2TestBase, _Http2ServerBase): +class TestWithBodies(_Http2Test): tmp_data_buffer_foobar = b'' - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() - @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): @@ -302,17 +299,7 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestPushPromise(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestPushPromise(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -465,17 +452,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestConnectionLost(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestConnectionLost(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -517,17 +494,7 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class(h2_server_settings={h2.settings.MAX_CONCURRENT_STREAMS: 2}) - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestMaxConcurrentStreams(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -583,17 +550,7 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestConnectionTerminated(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestConnectionTerminated(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): -- cgit v1.2.3 From 7c092552989a6f328bb5defad744cdd37a14e8ec Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 10 Jul 2016 20:07:43 +0200 Subject: http2: more coverage --- mitmproxy/protocol/http2.py | 54 +++++---- test/mitmproxy/test_protocol_http2.py | 220 +++++++++++++++++++++++++++++++--- 2 files changed, 231 insertions(+), 43 deletions(-) diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 4b57174b..27c2a664 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -78,7 +78,7 @@ class SafeH2Connection(connection.H2Connection): self.send_data(stream_id, frame_chunk) try: self.conn.send(self.data_to_send()) - except Exception as e: + except Exception as e: # pragma: no cover raise e finally: self.lock.release() @@ -142,9 +142,9 @@ class Http2Layer(base.Layer): self.streams[eid].timestamp_start = time.time() self.streams[eid].no_body = (event.stream_ended is not None) if event.priority_updated is not None: - self.streams[eid].priority_weight = event.priority_updated.weight - self.streams[eid].priority_depends_on = event.priority_updated.depends_on self.streams[eid].priority_exclusive = event.priority_updated.exclusive + self.streams[eid].priority_depends_on = event.priority_updated.depends_on + self.streams[eid].priority_weight = event.priority_updated.weight self.streams[eid].handled_priority_event = event.priority_updated self.streams[eid].start() elif isinstance(event, events.ResponseReceived): @@ -155,10 +155,13 @@ class Http2Layer(base.Layer): self.streams[eid].response_arrived.set() elif isinstance(event, events.DataReceived): if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit: - raise netlib.exceptions.HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit)) - self.streams[eid].data_queue.put(event.data) - self.streams[eid].queued_data_length += len(event.data) - source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) + self.streams[eid].zombie = time.time() + source_conn.h2.safe_reset_stream(event.stream_id, 0x7) + self.log("HTTP body too large. Limit is {}.".format(self.config.body_size_limit), "info") + else: + self.streams[eid].data_queue.put(event.data) + self.streams[eid].queued_data_length += len(event.data) + source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) elif isinstance(event, events.StreamEnded): self.streams[eid].timestamp_end = time.time() self.streams[eid].data_finished.set() @@ -206,6 +209,11 @@ class Http2Layer(base.Layer): self.streams[event.pushed_stream_id].request_data_finished.set() self.streams[event.pushed_stream_id].start() elif isinstance(event, events.PriorityUpdated): + if eid in self.streams and self.streams[eid].handled_priority_event is event: + # this event was already handled during stream creation + # HeadersFrame + Priority information as RequestReceived + return True + mapped_stream_id = event.stream_id if mapped_stream_id in self.streams and self.streams[mapped_stream_id].server_stream_id: # if the stream is already up and running and was sent to the server @@ -213,13 +221,9 @@ class Http2Layer(base.Layer): mapped_stream_id = self.streams[mapped_stream_id].server_stream_id if eid in self.streams: - if self.streams[eid].handled_priority_event is event: - # this event was already handled during stream creation - # HeadersFrame + Priority information as RequestReceived - return True - self.streams[eid].priority_weight = event.weight - self.streams[eid].priority_depends_on = event.depends_on self.streams[eid].priority_exclusive = event.exclusive + self.streams[eid].priority_depends_on = event.depends_on + self.streams[eid].priority_weight = event.weight with self.server_conn.h2.lock: self.server_conn.h2.prioritize( @@ -248,10 +252,12 @@ class Http2Layer(base.Layer): def _cleanup_streams(self): death_time = time.time() - 10 - for stream_id in self.streams.keys(): - zombie = self.streams[stream_id].zombie - if zombie and zombie <= death_time: - self.streams.pop(stream_id, None) + + zombie_streams = [(stream_id, stream) for stream_id, stream in list(self.streams.items()) if stream.zombie] + outdated_streams = [stream_id for stream_id, stream in zombie_streams if stream.zombie <= death_time] + + for stream_id in outdated_streams: # pragma: no cover + self.streams.pop(stream_id, None) def _kill_all_streams(self): for stream in self.streams.values(): @@ -296,7 +302,7 @@ class Http2Layer(base.Layer): return self._cleanup_streams() - except Exception as e: + except Exception as e: # pragma: no cover self.log(repr(e), "info") self.log(traceback.format_exc(), "debug") self._kill_all_streams() @@ -326,9 +332,9 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.no_body = False - self.priority_weight = None - self.priority_depends_on = None self.priority_exclusive = None + self.priority_depends_on = None + self.priority_weight = None self.handled_priority_event = None @property @@ -428,11 +434,11 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.server_stream_id, headers, end_stream=self.no_body, - priority_weight=self.priority_weight, - priority_depends_on=self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on), priority_exclusive=self.priority_exclusive, + priority_depends_on=self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on), + priority_weight=self.priority_weight, ) - except Exception as e: + except Exception as e: # pragma: no cover raise e finally: self.server_conn.h2.lock.release() @@ -523,7 +529,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) try: layer() - except exceptions.ProtocolException as e: + except exceptions.ProtocolException as e: # pragma: no cover self.log(repr(e), "info") self.log(traceback.format_exc(), "debug") diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index a4f6b574..cb7cebca 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -60,7 +60,10 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): except HttpException: print(traceback.format_exc()) assert False + except netlib.exceptions.TcpDisconnect: + break except: + print(traceback.format_exc()) break self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() @@ -70,8 +73,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): done = True break + except netlib.exceptions.TcpDisconnect: + done = True except: done = True + print(traceback.format_exc()) break def handle_server_event(self, h2_conn, rfile, wfile): @@ -138,11 +144,22 @@ class _Http2TestBase(object): return client, h2_conn - def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], body=b''): + def _send_request(self, + wfile, + h2_conn, + stream_id=1, + headers=[], + body=b'', + priority_exclusive=None, + priority_depends_on=None, + priority_weight=None): h2_conn.send_headers( stream_id=stream_id, headers=headers, end_stream=(len(body) == 0), + priority_exclusive=priority_exclusive, + priority_depends_on=priority_depends_on, + priority_weight=priority_weight, ) if body: h2_conn.send_data(stream_id, body) @@ -166,6 +183,7 @@ class _Http2Test(_Http2TestBase, _Http2ServerBase): @requires_alpn class TestSimple(_Http2Test): + request_body_buffer = b'' @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -190,13 +208,16 @@ class TestSimple(_Http2Test): ('föo', 'bär'), ('X-Stream-ID', str(event.stream_id)), ]) - h2_conn.send_data(event.stream_id, b'foobar') + h2_conn.send_data(event.stream_id, b'response body') h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() + elif isinstance(event, h2.events.DataReceived): + self.request_body_buffer += event.data return True def test_simple(self): + response_body_buffer = b'' client, h2_conn = self._setup_connection() self._send_request( @@ -210,7 +231,7 @@ class TestSimple(_Http2Test): ('ClIeNt-FoO', 'client-bar-1'), ('ClIeNt-FoO', 'client-bar-2'), ], - body=b'my request body echoed back to me') + body=b'request body') done = False while not done: @@ -225,7 +246,9 @@ class TestSimple(_Http2Test): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.DataReceived): + response_body_buffer += event.data + elif isinstance(event, h2.events.StreamEnded): done = True h2_conn.close_connection() @@ -236,31 +259,41 @@ class TestSimple(_Http2Test): assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' assert self.master.state.flows[0].response.headers['föo'] == 'bär' - assert self.master.state.flows[0].response.body == b'foobar' + assert self.master.state.flows[0].response.body == b'response body' + assert self.request_body_buffer == b'request body' + assert response_body_buffer == b'response body' @requires_alpn -class TestWithBodies(_Http2Test): - tmp_data_buffer_foobar = b'' +class TestRequestWithPriority(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - if isinstance(event, h2.events.DataReceived): - self.tmp_data_buffer_foobar += event.data - elif isinstance(event, h2.events.StreamEnded): - h2_conn.send_headers(1, [ - (':status', '200'), - ]) - h2_conn.send_data(1, self.tmp_data_buffer_foobar) - h2_conn.end_stream(1) + elif isinstance(event, h2.events.RequestReceived): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + if event.priority_updated: + headers.append(('priority_exclusive', event.priority_updated.exclusive)) + headers.append(('priority_depends_on', event.priority_updated.depends_on)) + headers.append(('priority_weight', event.priority_updated.weight)) + h2_conn.send_headers(event.stream_id, headers) + h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() - return True - def test_with_bodies(self): + def test_request_with_priority(self): client, h2_conn = self._setup_connection() self._send_request( @@ -272,7 +305,9 @@ class TestWithBodies(_Http2Test): (':scheme', 'https'), (':path', '/'), ], - body=b'foobar with request body', + priority_exclusive = True, + priority_depends_on = 42424242, + priority_weight = 42, ) done = False @@ -295,7 +330,149 @@ class TestWithBodies(_Http2Test): client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() - assert self.master.state.flows[0].response.body == b'foobar with request body' + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response.headers['priority_exclusive'] == 'True' + assert self.master.state.flows[0].response.headers['priority_depends_on'] == '42424242' + assert self.master.state.flows[0].response.headers['priority_weight'] == '42' + + def test_request_without_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert 'priority_exclusive' not in self.master.state.flows[0].response.headers + assert 'priority_depends_on' not in self.master.state.flows[0].response.headers + assert 'priority_weight' not in self.master.state.flows[0].response.headers + + +@requires_alpn +class TestStreamResetFromServer(_Http2Test): + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + h2_conn.reset_stream(event.stream_id, 0x8) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_request_with_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamReset): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response is None + + +@requires_alpn +class TestBodySizeLimit(_Http2Test): + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + return True + + def test_body_size_limit(self): + self.config.body_size_limit = 20 + + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + body=b'very long body over 20 characters long', + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamReset): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 0 @requires_alpn @@ -496,6 +673,11 @@ class TestConnectionLost(_Http2Test): @requires_alpn class TestMaxConcurrentStreams(_Http2Test): + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class(h2_server_settings={h2.settings.MAX_CONCURRENT_STREAMS: 2}) + @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): -- cgit v1.2.3 From 62176142043efb814608c61edf1e18f91eb7a13f Mon Sep 17 00:00:00 2001 From: strohu Date: Tue, 12 Jul 2016 13:18:58 +0100 Subject: Fix is_ascii check when determining next layer --- mitmproxy/proxy/root_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 57183c7e..4d6509d4 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -100,7 +100,7 @@ class RootContext(object): is_ascii = ( len(d) == 3 and # expect A-Za-z - all(65 <= x <= 90 and 97 <= x <= 122 for x in six.iterbytes(d)) + all(65 <= x <= 90 or 97 <= x <= 122 for x in six.iterbytes(d)) ) if self.config.rawtcp and not is_ascii: return protocol.RawTCPLayer(top_layer) -- cgit v1.2.3 From 6a9407d7cc4ac5555180a2ee331ff95eef131902 Mon Sep 17 00:00:00 2001 From: strohu Date: Tue, 12 Jul 2016 15:47:01 +0100 Subject: Make sudo pfctl error check Python 3 compatible In Python 3, subprocess.check_output() returns a sequence of bytes. This change ensures that it will be converted to a string, so the substring test for the sudo error message does not raise a TypeError. This fixes the code in Python 3 while remaining compatible with Python 2. --- mitmproxy/platform/osx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/platform/osx.py b/mitmproxy/platform/osx.py index b5dce793..6a555f32 100644 --- a/mitmproxy/platform/osx.py +++ b/mitmproxy/platform/osx.py @@ -23,12 +23,12 @@ class Resolver(object): try: stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - if "sudo: a password is required" in e.output: + if "sudo: a password is required" in e.output.decode(errors="replace"): insufficient_priv = True else: raise RuntimeError("Error getting pfctl state: " + repr(e)) else: - insufficient_priv = "sudo: a password is required" in stxt + insufficient_priv = "sudo: a password is required" in stxt.decode(errors="replace") if insufficient_priv: raise RuntimeError( -- cgit v1.2.3 From 3579c6dd43529595ce1a37d96f19a446419c6ee5 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 12 Jul 2016 19:55:28 +0200 Subject: http2: fix race-condition in tests --- test/mitmproxy/test_protocol_http2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index cb7cebca..a100ac2d 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -192,7 +192,7 @@ class TestSimple(_Http2Test): elif isinstance(event, h2.events.RequestReceived): assert (b'client-foo', b'client-bar-1') in event.headers assert (b'client-foo', b'client-bar-2') in event.headers - + elif isinstance(event, h2.events.StreamEnded): import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: -- cgit v1.2.3 From 2624911d75670afaff8631943d567bfa2b42d7b8 Mon Sep 17 00:00:00 2001 From: Clemens Brunner Date: Tue, 12 Jul 2016 23:52:33 +0200 Subject: fixed bug (#1342) fix minor mitmweb issues --- web/src/js/components/Header/FlowMenu.jsx | 4 ++-- web/src/js/ducks/ui.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/src/js/components/Header/FlowMenu.jsx b/web/src/js/components/Header/FlowMenu.jsx index 8d13dd6a..9855cde3 100644 --- a/web/src/js/components/Header/FlowMenu.jsx +++ b/web/src/js/components/Header/FlowMenu.jsx @@ -15,11 +15,11 @@ function FlowMenu({ flow, acceptFlow, replayFlow, duplicateFlow, removeFlow, rev return (
-
diff --git a/web/src/js/ducks/ui.js b/web/src/js/ducks/ui.js index f8234fdb..4de460aa 100644 --- a/web/src/js/ducks/ui.js +++ b/web/src/js/ducks/ui.js @@ -13,8 +13,8 @@ export default function reducer(state = defaultState, action) { activeMenu: action.activeMenu } case SELECT: - let isNewSelect = (action.flowId && !action.currentSelection) - let isDeselect = (!action.flowId && action.currentSelection) + let isNewSelect = (action.id && !action.currentSelection) + let isDeselect = (!action.id && action.currentSelection) if(isNewSelect) { return { ...state, -- cgit v1.2.3 From a20f8e9620c0cfcb40500113cbeb813a05a195bb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 18:45:50 +1200 Subject: More powerful Options scheme This prepares us for the addon configuration mechanism and gives us a more flexible way to handle options changes. This changeset should spell the end of the current anti-pattern in our codebase where we duplicate data out of options onto the master when mutability is needed. From now on, Options can be the one source of thruth. - Change notifications - Rollback on error --- mitmproxy/console/master.py | 10 ++----- mitmproxy/dump.py | 10 ++----- mitmproxy/exceptions.py | 4 +++ mitmproxy/options.py | 66 +++++++++++++++++++++++++++++++++++++++++ mitmproxy/web/master.py | 11 ++----- test/mitmproxy/test_options.py | 67 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 25 deletions(-) create mode 100644 mitmproxy/options.py create mode 100644 test/mitmproxy/test_options.py diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 93b5766d..99af0722 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -20,6 +20,7 @@ from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import script +import mitmproxy.options from mitmproxy.console import flowlist from mitmproxy.console import flowview from mitmproxy.console import grideditor @@ -175,7 +176,7 @@ class ConsoleState(flow.State): self.add_flow_setting(flow, "marked", marked) -class Options(object): +class Options(mitmproxy.options.Options): attributes = [ "app", "app_domain", @@ -210,13 +211,6 @@ class Options(object): "outfile", ] - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - for i in self.attributes: - if not hasattr(self, i): - setattr(self, i, None) - class ConsoleMaster(flow.FlowMaster): palette = [] diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 10465b63..bfefb319 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -11,6 +11,7 @@ from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import flow +from mitmproxy import options from netlib import human from netlib import tcp from netlib import strutils @@ -20,7 +21,7 @@ class DumpError(Exception): pass -class Options(object): +class Options(options.Options): attributes = [ "app", "app_host", @@ -53,13 +54,6 @@ class Options(object): "replay_ignore_host" ] - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - for i in self.attributes: - if not hasattr(self, i): - setattr(self, i, None) - class DumpMaster(flow.FlowMaster): diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 63bd8d3d..282784b6 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -95,3 +95,7 @@ class FlowReadException(ProxyException): class ControlException(ProxyException): pass + + +class OptionsError(Exception): + pass diff --git a/mitmproxy/options.py b/mitmproxy/options.py new file mode 100644 index 00000000..7389df1f --- /dev/null +++ b/mitmproxy/options.py @@ -0,0 +1,66 @@ +from __future__ import absolute_import, print_function, division + +import contextlib +import blinker +import pprint + +from mitmproxy import exceptions + + +class Options(object): + """ + .changed is a blinker Signal that triggers whenever options are + updated. If any handler in the chain raises an exceptions.OptionsError + exception, all changes are rolled back, the exception is suppressed, + and the .errored signal is notified. + """ + attributes = [] + + def __init__(self, **kwargs): + self.__dict__["changed"] = blinker.Signal() + self.__dict__["errored"] = blinker.Signal() + self.__dict__["_opts"] = dict([(i, None) for i in self.attributes]) + for k, v in kwargs.items(): + self._opts[k] = v + + @contextlib.contextmanager + def rollback(self): + old = self._opts.copy() + try: + yield + except exceptions.OptionsError as e: + # Notify error handlers + self.errored.send(self, exc=e) + # Rollback + self.__dict__["_opts"] = old + self.changed.send(self) + + def __eq__(self, other): + return self._opts == other._opts + + def __copy__(self): + return self.__class__(**self._opts) + + def __getattr__(self, attr): + return self._opts[attr] + + def __setattr__(self, attr, value): + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + with self.rollback(): + self._opts[attr] = value + self.changed.send(self) + + def get(self, k, d=None): + return self._opts.get(k, d) + + def update(self, **kwargs): + for k in kwargs: + if k not in self._opts: + raise KeyError("No such option: %s" % k) + with self.rollback(): + self._opts.update(kwargs) + self.changed.send(self) + + def __repr__(self): + return pprint.pformat(self._opts) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 737bb95f..2b55e74e 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -9,6 +9,7 @@ import tornado.ioloop from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow +from mitmproxy import options from mitmproxy.web import app from netlib.http import authentication @@ -88,7 +89,7 @@ class WebState(flow.State): ) -class Options(object): +class Options(options.Options): attributes = [ "app", "app_domain", @@ -124,14 +125,6 @@ class Options(object): "wsingleuser", "whtpasswd", ] - - def __init__(self, **kwargs): - for k, v in kwargs.items(): - setattr(self, k, v) - for i in self.attributes: - if not hasattr(self, i): - setattr(self, i, None) - def process_web_options(self, parser): if self.wsingleuser or self.whtpasswd: if self.wsingleuser: diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py new file mode 100644 index 00000000..5fdb7abe --- /dev/null +++ b/test/mitmproxy/test_options.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import, print_function, division +import copy + +from mitmproxy import options +from mitmproxy import exceptions +from netlib.tutils import raises + + +class TO(options.Options): + attributes = [ + "one", + "two" + ] + + +def test_options(): + o = TO(two="three") + assert o.one is None + assert o.two == "three" + o.one = "one" + assert o.one == "one" + raises("no such option", setattr, o, "nonexistent", "value") + raises("no such option", o.update, nonexistent = "value") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + o.changed.connect(sub) + + o.one = "ninety" + assert len(rec) == 1 + assert rec[-1].one == "ninety" + + o.update(one="oink") + assert len(rec) == 2 + assert rec[-1].one == "oink" + + +def test_rollback(): + o = TO(one="two") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + recerr = [] + + def errsub(opts, **kwargs): + recerr.append(kwargs) + + def err(opts): + if opts.one == "ten": + raise exceptions.OptionsError + + o.changed.connect(sub) + o.changed.connect(err) + o.errored.connect(errsub) + + o.one = "ten" + assert isinstance(recerr[0]["exc"], exceptions.OptionsError) + assert o.one == "two" + assert len(rec) == 2 + assert rec[0].one == "ten" + assert rec[1].one == "two" -- cgit v1.2.3 From c9a0fe6a0e6d70fa2aea1b8dc337609a9439ade1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 19:05:32 +1200 Subject: Show how options integrates with console This is not functional at the moment, because all mutable options are still on master. --- mitmproxy/console/master.py | 8 ++++++++ mitmproxy/console/options.py | 1 + mitmproxy/console/statusbar.py | 1 + mitmproxy/web/master.py | 1 + 4 files changed, 11 insertions(+) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 99af0722..1daf1127 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -217,8 +217,10 @@ class ConsoleMaster(flow.FlowMaster): def __init__(self, server, options): flow.FlowMaster.__init__(self, server, ConsoleState()) + self.stream_path = None self.options = options + self.options.errored.connect(self.options_error) if options.replacements: for i in options.replacements: @@ -298,6 +300,12 @@ class ConsoleMaster(flow.FlowMaster): self.__dict__[name] = value signals.update_settings.send(self) + def options_error(self, opts, exc): + signals.status_message.send( + message=str(exc), + expire=1 + ) + def load_script(self, command, use_reloader=True): # We default to using the reloader in the console ui. return super(ConsoleMaster, self).load_script(command, use_reloader) diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index 5a01c9d5..6a4b8dd6 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -140,6 +140,7 @@ class Options(urwid.WidgetWrap): ) self.master.loop.widget.footer.update("") signals.update_settings.connect(self.sig_update_settings) + master.options.changed.connect(self.sig_update_settings) def sig_update_settings(self, sender): self.lb.walker._modified() diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index e576b565..d1ab5906 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -122,6 +122,7 @@ class StatusBar(urwid.WidgetWrap): self._w = urwid.Pile([self.ib, self.ab]) signals.update_settings.connect(self.sig_update_settings) signals.flowlist_change.connect(self.sig_update_settings) + master.options.changed.connect(self.sig_update_settings) self.redraw() def sig_update_settings(self, sender): diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 2b55e74e..008b74f8 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -125,6 +125,7 @@ class Options(options.Options): "wsingleuser", "whtpasswd", ] + def process_web_options(self, parser): if self.wsingleuser or self.whtpasswd: if self.wsingleuser: -- cgit v1.2.3 From 077850bd107d7ffe1cf3d4a8667bb04ea47beb96 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 20:05:17 +1200 Subject: Raise AttributeError from Options.__getattr__ --- mitmproxy/options.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 7389df1f..0cc5fee1 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -42,7 +42,10 @@ class Options(object): return self.__class__(**self._opts) def __getattr__(self, attr): - return self._opts[attr] + if attr in self._opts: + return self._opts[attr] + else: + raise AttributeError() def __setattr__(self, attr, value): if attr not in self._opts: -- cgit v1.2.3 From 7f8fd3cdffedb537f95773110d8ef2be60666133 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 23:26:04 +1200 Subject: Basic outline of addons Add addons.py, integrate with our event mechanism, and change the Master API so options is the first init argument. --- mitmproxy/addons.py | 63 +++++++++++++++++++++++++++++++++++++++ mitmproxy/console/master.py | 3 +- mitmproxy/controller.py | 11 ++++++- mitmproxy/dump.py | 22 +++++++------- mitmproxy/flow/master.py | 4 +-- mitmproxy/options.py | 5 +++- mitmproxy/web/master.py | 7 +++-- test/mitmproxy/test_addons.py | 20 +++++++++++++ test/mitmproxy/test_controller.py | 4 +-- test/mitmproxy/test_flow.py | 38 +++++++++++------------ test/mitmproxy/test_script.py | 2 +- test/mitmproxy/tservers.py | 2 +- 12 files changed, 138 insertions(+), 43 deletions(-) create mode 100644 mitmproxy/addons.py create mode 100644 test/mitmproxy/test_addons.py diff --git a/mitmproxy/addons.py b/mitmproxy/addons.py new file mode 100644 index 00000000..7ac65a09 --- /dev/null +++ b/mitmproxy/addons.py @@ -0,0 +1,63 @@ +from __future__ import absolute_import, print_function, division +from mitmproxy import exceptions +import pprint + + +def _get_name(itm): + return getattr(itm, "name", itm.__class__.__name__) + + +class Addons(object): + def __init__(self, master): + self.chain = [] + self.master = master + master.options.changed.connect(self.options_update) + + def options_update(self, options): + for i in self.chain: + with self.master.handlecontext(): + i.configure(options) + + def add(self, *addons): + self.chain.extend(addons) + for i in addons: + self.invoke_with_context(i, "configure", self.master.options) + + def remove(self, addon): + self.chain = [i for i in self.chain if i is not addon] + self.invoke_with_context(addon, "done") + + def done(self): + for i in self.chain: + self.invoke_with_context(i, "done") + + def has_addon(self, name): + """ + Is an addon with this name registered? + """ + for i in self.chain: + if _get_name(i) == name: + return True + + def __len__(self): + return len(self.chain) + + def __str__(self): + return pprint.pformat([str(i) for i in self.chain]) + + def invoke_with_context(self, addon, name, *args, **kwargs): + with self.master.handlecontext(): + self.invoke(addon, name, *args, **kwargs) + + def invoke(self, addon, name, *args, **kwargs): + func = getattr(addon, name, None) + if func: + if not callable(func): + raise exceptions.AddonError( + "Addon handler %s not callable" % name + ) + func(*args, **kwargs) + + def __call__(self, name, *args, **kwargs): + for i in self.chain: + self.invoke(i, name, *args, **kwargs) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 1daf1127..00905f36 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -216,10 +216,9 @@ class ConsoleMaster(flow.FlowMaster): palette = [] def __init__(self, server, options): - flow.FlowMaster.__init__(self, server, ConsoleState()) + flow.FlowMaster.__init__(self, options, server, ConsoleState()) self.stream_path = None - self.options = options self.options.errored.connect(self.options_error) if options.replacements: diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index e2be3a53..d09038f8 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -6,6 +6,8 @@ import contextlib from six.moves import queue +from mitmproxy import addons +from mitmproxy import options from . import ctx as mitmproxy_ctx from netlib import basethread from . import exceptions @@ -49,7 +51,9 @@ class Master(object): """ The master handles mitmproxy's main event loop. """ - def __init__(self, *servers): + def __init__(self, opts, *servers): + self.options = opts or options.Options() + self.addons = addons.Addons(self) self.event_queue = queue.Queue() self.should_exit = threading.Event() self.servers = [] @@ -121,6 +125,7 @@ class Master(object): for server in self.servers: server.shutdown() self.should_exit.set() + self.addons.done() class ServerThread(basethread.BaseThread): @@ -191,6 +196,10 @@ def handler(f): with master.handlecontext(): ret = f(master, message) + if handling: + # Python2/3 compatibility hack + fn = getattr(f, "func_name", None) or getattr(f, "__name__") + master.addons(fn) if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index bfefb319..cd515945 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -58,7 +58,7 @@ class Options(options.Options): class DumpMaster(flow.FlowMaster): def __init__(self, server, options, outfile=None): - flow.FlowMaster.__init__(self, server, flow.State()) + flow.FlowMaster.__init__(self, options, server, flow.State()) self.outfile = outfile self.o = options self.anticache = options.anticache @@ -137,8 +137,8 @@ class DumpMaster(flow.FlowMaster): self.add_event("Flow file corrupted.", "error") raise DumpError(v) - if self.o.app: - self.start_app(self.o.app_host, self.o.app_port) + if self.options.app: + self.start_app(self.options.app_host, self.options.app_port) def _readflow(self, paths): """ @@ -152,7 +152,7 @@ class DumpMaster(flow.FlowMaster): def add_event(self, e, level="info"): needed = dict(error=0, info=1, debug=2).get(level, 1) - if self.o.verbosity >= needed: + if self.options.verbosity >= needed: self.echo( e, fg="red" if level == "error" else None, @@ -172,7 +172,7 @@ class DumpMaster(flow.FlowMaster): click.secho(text, file=self.outfile, **style) def _echo_message(self, message): - if self.o.flow_detail >= 2 and hasattr(message, "headers"): + if self.options.flow_detail >= 2 and hasattr(message, "headers"): headers = "\r\n".join( "{}: {}".format( click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True), @@ -180,7 +180,7 @@ class DumpMaster(flow.FlowMaster): for k, v in message.headers.fields ) self.echo(headers, indent=4) - if self.o.flow_detail >= 3: + if self.options.flow_detail >= 3: if message.content is None: self.echo("(content missing)", indent=4) elif message.content: @@ -213,7 +213,7 @@ class DumpMaster(flow.FlowMaster): for (style, text) in line: yield click.style(text, **styles.get(style, {})) - if self.o.flow_detail == 3: + if self.options.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines @@ -228,7 +228,7 @@ class DumpMaster(flow.FlowMaster): if next(lines, None): self.echo("(cut off)", indent=4, dim=True) - if self.o.flow_detail >= 2: + if self.options.flow_detail >= 2: self.echo("") def _echo_request_line(self, flow): @@ -302,7 +302,7 @@ class DumpMaster(flow.FlowMaster): self.echo(line) def echo_flow(self, f): - if self.o.flow_detail == 0: + if self.options.flow_detail == 0: return if f.request: @@ -350,7 +350,7 @@ class DumpMaster(flow.FlowMaster): def tcp_message(self, f): super(DumpMaster, self).tcp_message(f) - if self.o.flow_detail == 0: + if self.options.flow_detail == 0: return message = f.messages[-1] direction = "->" if message.from_client else "<-" @@ -362,7 +362,7 @@ class DumpMaster(flow.FlowMaster): self._echo_message(message) def run(self): # pragma: no cover - if self.o.rfile and not self.o.keepserving: + if self.options.rfile and not self.options.keepserving: self.unload_scripts() # make sure to trigger script unload events. return super(DumpMaster, self).run() diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 7590a3fa..b1951f94 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -27,8 +27,8 @@ class FlowMaster(controller.Master): if len(self.servers) > 0: return self.servers[0] - def __init__(self, server, state): - super(FlowMaster, self).__init__() + def __init__(self, options, server, state): + super(FlowMaster, self).__init__(options) if server: self.add_server(server) self.state = state diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 7389df1f..0cc5fee1 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -42,7 +42,10 @@ class Options(object): return self.__class__(**self._opts) def __getattr__(self, attr): - return self._opts[attr] + if attr in self._opts: + return self._opts[attr] + else: + raise AttributeError() def __setattr__(self, attr, value): if attr not in self._opts: diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 008b74f8..22972c14 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -147,9 +147,10 @@ class Options(options.Options): class WebMaster(flow.FlowMaster): def __init__(self, server, options): - self.options = options - super(WebMaster, self).__init__(server, WebState()) - self.app = app.Application(self, self.options.wdebug, self.options.wauthenticator) + super(WebMaster, self).__init__(options, server, WebState()) + self.app = app.Application( + self, self.options.wdebug, self.options.wauthenticator + ) if options.rfile: try: self.load_flows_file(options.rfile) diff --git a/test/mitmproxy/test_addons.py b/test/mitmproxy/test_addons.py new file mode 100644 index 00000000..1861d4ac --- /dev/null +++ b/test/mitmproxy/test_addons.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, print_function, division +from mitmproxy import addons +from mitmproxy import controller +from mitmproxy import options + + +class TAddon: + def __init__(self, name): + self.name = name + + def __repr__(self): + return "Addon(%s)" % self.name + + +def test_simple(): + m = controller.Master(options.Options()) + a = addons.Addons(m) + a.add(TAddon("one")) + assert a.has_addon("one") + assert not a.has_addon("two") diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 5a68e15b..6d4b8fe6 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -25,7 +25,7 @@ class TestMaster(object): # Speed up test super(DummyMaster, self).tick(0) - m = DummyMaster() + m = DummyMaster(None) assert not m.should_exit.is_set() msg = TMsg() msg.reply = controller.DummyReply() @@ -34,7 +34,7 @@ class TestMaster(object): assert m.should_exit.is_set() def test_server_simple(self): - m = controller.Master() + m = controller.Master(None) s = DummyServer(None) m.add_server(s) m.start() diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 74b3f599..eda01ad9 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -139,7 +139,7 @@ class TestClientPlaybackState: def test_tick(self): first = tutils.tflow() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.start_client_playback([first, tutils.tflow()], True) c = fm.client_playback c.testing = True @@ -470,7 +470,7 @@ class TestFlow(object): def test_kill(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(mock.Mock()) f.kill(fm) @@ -479,7 +479,7 @@ class TestFlow(object): def test_killall(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(fm) @@ -714,7 +714,7 @@ class TestSerialize: def test_load_flows(self): r = self._treader() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_flows(r) assert len(s.flows) == 6 @@ -725,7 +725,7 @@ class TestSerialize: mode="reverse", upstream_server=("https", ("use-this-domain", 80)) ) - fm = flow.FlowMaster(DummyServer(conf), s) + fm = flow.FlowMaster(None, DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" @@ -772,7 +772,7 @@ class TestFlowMaster: def test_load_script(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_script(tutils.test_data.path("data/scripts/a.py")) fm.load_script(tutils.test_data.path("data/scripts/a.py")) @@ -788,14 +788,14 @@ class TestFlowMaster: def test_getset_ignore(self): p = mock.Mock() p.config.check_ignore = HostMatcher() - fm = flow.FlowMaster(p, flow.State()) + fm = flow.FlowMaster(None, p, flow.State()) assert not fm.get_ignore_filter() fm.set_ignore_filter(["^apple\.com:", ":443$"]) assert fm.get_ignore_filter() def test_replay(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) f.request.content = None assert "missing" in fm.replay_request(f) @@ -808,7 +808,7 @@ class TestFlowMaster: def test_script_reqerr(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_script(tutils.test_data.path("data/scripts/reqerr.py")) f = tutils.tflow() fm.clientconnect(f.client_conn) @@ -816,7 +816,7 @@ class TestFlowMaster: def test_script(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_script(tutils.test_data.path("data/scripts/all.py")) f = tutils.tflow(resp=True) @@ -852,7 +852,7 @@ class TestFlowMaster: def test_duplicate_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) fm.load_flow(f) assert s.flow_count() == 1 @@ -863,12 +863,12 @@ class TestFlowMaster: def test_create_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) assert fm.create_request("GET", "http", "example.com", 80, "/") def test_all(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.anticache = True fm.anticomp = True f = tutils.tflow(req=None) @@ -895,7 +895,7 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) + fm = flow.FlowMaster(None, DummyServer(ProxyConfig()), s) assert not fm.start_server_playback( pb, False, @@ -923,7 +923,7 @@ class TestFlowMaster: f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) @@ -965,7 +965,7 @@ class TestFlowMaster: f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.refresh_server_playback = True fm.start_server_playback( pb, @@ -985,7 +985,7 @@ class TestFlowMaster: def test_stickycookie(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) assert "Invalid" in fm.set_stickycookie("~h") fm.set_stickycookie(".*") assert fm.stickycookie_state @@ -1007,7 +1007,7 @@ class TestFlowMaster: def test_stickyauth(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) assert "Invalid" in fm.set_stickyauth("~h") fm.set_stickyauth(".*") assert fm.stickyauth_state @@ -1035,7 +1035,7 @@ class TestFlowMaster: return list(r.stream()) s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) with open(p, "ab") as tmpfile: diff --git a/test/mitmproxy/test_script.py b/test/mitmproxy/test_script.py index 81994780..1e8220f1 100644 --- a/test/mitmproxy/test_script.py +++ b/test/mitmproxy/test_script.py @@ -4,7 +4,7 @@ from . import tutils def test_duplicate_flow(): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_script(tutils.test_data.path("data/scripts/duplicate_flow.py")) f = tutils.tflow() fm.request(f) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 0760cb53..69a50b9d 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -34,7 +34,7 @@ class TestMaster(flow.FlowMaster): config.port = 0 s = ProxyServer(config) state = flow.State() - flow.FlowMaster.__init__(self, s, state) + flow.FlowMaster.__init__(self, None, s, state) self.apps.add(testapp, "testapp", 80) self.apps.add(errapp, "errapp", 80) self.clear_log() -- cgit v1.2.3 From 255e1eb00b76f169305d7c4ae3bba91403f3f924 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 23:45:49 +1200 Subject: Add the addons mechanism Demonstrate how it works and interacts with Options by adding our first new builtin: anticomp. --- mitmproxy/builtins/__init__.py | 9 +++++++++ mitmproxy/builtins/anticomp.py | 12 ++++++++++++ mitmproxy/console/master.py | 3 ++- mitmproxy/console/options.py | 4 ++-- mitmproxy/console/statusbar.py | 2 +- mitmproxy/controller.py | 2 +- mitmproxy/dump.py | 2 ++ mitmproxy/flow/master.py | 3 --- mitmproxy/web/master.py | 2 ++ test/mitmproxy/builtins/__init__.py | 0 test/mitmproxy/builtins/test_anticomp.py | 22 ++++++++++++++++++++++ test/mitmproxy/mastertest.py | 8 +++++++- test/mitmproxy/test_flow.py | 1 - 13 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 mitmproxy/builtins/__init__.py create mode 100644 mitmproxy/builtins/anticomp.py create mode 100644 test/mitmproxy/builtins/__init__.py create mode 100644 test/mitmproxy/builtins/test_anticomp.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py new file mode 100644 index 00000000..59173145 --- /dev/null +++ b/mitmproxy/builtins/__init__.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import, print_function, division + +from mitmproxy.builtins import anticomp + + +def default_addons(): + return [ + anticomp.AntiComp(), + ] diff --git a/mitmproxy/builtins/anticomp.py b/mitmproxy/builtins/anticomp.py new file mode 100644 index 00000000..2820a85c --- /dev/null +++ b/mitmproxy/builtins/anticomp.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, print_function, division + +class AntiComp: + def __init__(self): + self.enabled = False + + def configure(self, options): + self.enabled = options.anticomp + + def request(self, flow): + if self.enabled: + flow.request.anticomp() diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 00905f36..06a05464 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -15,6 +15,7 @@ import weakref import urwid +from mitmproxy import builtins from mitmproxy import contentviews from mitmproxy import controller from mitmproxy import exceptions @@ -217,6 +218,7 @@ class ConsoleMaster(flow.FlowMaster): def __init__(self, server, options): flow.FlowMaster.__init__(self, options, server, ConsoleState()) + self.addons.add(*builtins.default_addons()) self.stream_path = None self.options.errored.connect(self.options_error) @@ -251,7 +253,6 @@ class ConsoleMaster(flow.FlowMaster): self.refresh_server_playback = options.refresh_server_playback self.anticache = options.anticache - self.anticomp = options.anticomp self.killextra = options.kill self.rheaders = options.rheaders self.nopop = options.nopop diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index 6a4b8dd6..db6d405a 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -102,7 +102,7 @@ class Options(urwid.WidgetWrap): select.Option( "Anti-Compression", "o", - lambda: master.anticomp, + lambda: master.options.anticomp, self.toggle_anticomp ), select.Option( @@ -177,7 +177,7 @@ class Options(urwid.WidgetWrap): self.master.anticache = not self.master.anticache def toggle_anticomp(self): - self.master.anticomp = not self.master.anticomp + self.master.options.anticomp = not self.master.options.anticomp def toggle_killextra(self): self.master.killextra = not self.master.killextra diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index d1ab5906..543c6771 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -189,7 +189,7 @@ class StatusBar(urwid.WidgetWrap): opts = [] if self.master.anticache: opts.append("anticache") - if self.master.anticomp: + if self.master.options.anticomp: opts.append("anticomp") if self.master.showhost: opts.append("showhost") diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index d09038f8..8b968eb5 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -199,7 +199,7 @@ def handler(f): if handling: # Python2/3 compatibility hack fn = getattr(f, "func_name", None) or getattr(f, "__name__") - master.addons(fn) + master.addons(fn, message) if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index cd515945..cf7aa5c9 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -12,6 +12,7 @@ from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import flow from mitmproxy import options +from mitmproxy import builtins from netlib import human from netlib import tcp from netlib import strutils @@ -59,6 +60,7 @@ class DumpMaster(flow.FlowMaster): def __init__(self, server, options, outfile=None): flow.FlowMaster.__init__(self, options, server, flow.State()) + self.addons.add(*builtins.default_addons()) self.outfile = outfile self.o = options self.anticache = options.anticache diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index b1951f94..ed2ee138 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -46,7 +46,6 @@ class FlowMaster(controller.Master): self.stickyauth_txt = None self.anticache = False - self.anticomp = False self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] self.refresh_server_playback = False self.replacehooks = modules.ReplaceHooks() @@ -332,8 +331,6 @@ class FlowMaster(controller.Master): if self.anticache: f.request.anticache() - if self.anticomp: - f.request.anticomp() if self.server_playback: pb = self.do_server_playback(f) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 22972c14..7c775c32 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -6,6 +6,7 @@ import collections import tornado.httpserver import tornado.ioloop +from mitmproxy import builtins from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow @@ -148,6 +149,7 @@ class WebMaster(flow.FlowMaster): def __init__(self, server, options): super(WebMaster, self).__init__(options, server, WebState()) + self.addons.add(*builtins.default_addons()) self.app = app.Application( self, self.options.wdebug, self.options.wauthenticator ) diff --git a/test/mitmproxy/builtins/__init__.py b/test/mitmproxy/builtins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/mitmproxy/builtins/test_anticomp.py b/test/mitmproxy/builtins/test_anticomp.py new file mode 100644 index 00000000..6bfd54bb --- /dev/null +++ b/test/mitmproxy/builtins/test_anticomp.py @@ -0,0 +1,22 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticomp +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options + + +class TestAntiComp(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticomp = True), None, s) + sa = anticomp.AntiComp() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + + f.request.headers["Accept-Encoding"] = "foobar" + self.invoke(m, "request", f) + assert f.request.headers["Accept-Encoding"] == "identity" diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 9e726a32..06854e25 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -3,10 +3,16 @@ import mock from . import tutils import netlib.tutils -from mitmproxy import flow, proxy, models +from mitmproxy import flow, proxy, models, controller class MasterTest: + def invoke(self, master, handler, message): + with master.handlecontext(): + func = getattr(master, handler) + func(message) + message.reply = controller.DummyReply() + def cycle(self, master, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) l = proxy.Log("connect") diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index eda01ad9..6b270872 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -870,7 +870,6 @@ class TestFlowMaster: s = flow.State() fm = flow.FlowMaster(None, None, s) fm.anticache = True - fm.anticomp = True f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) f.request = HTTPRequest.wrap(netlib.tutils.treq()) -- cgit v1.2.3 From 9ab1db513fce2c638e8deb6b6d0609d4b50b7bbb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 13 Jul 2016 23:50:25 +1200 Subject: Satisfy our ruthless linting overlords --- mitmproxy/builtins/anticomp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/builtins/anticomp.py b/mitmproxy/builtins/anticomp.py index 2820a85c..50bd1b73 100644 --- a/mitmproxy/builtins/anticomp.py +++ b/mitmproxy/builtins/anticomp.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, print_function, division + class AntiComp: def __init__(self): self.enabled = False -- cgit v1.2.3 From 696789b8ec41c6189226896c93eb121014946eff Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 09:33:26 +1200 Subject: func.__name__ works in both 2.7 and 3.5 Thanks to @resam and @dwfreed for pointing this out --- mitmproxy/controller.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 8b968eb5..2f0c8bf2 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -197,9 +197,7 @@ def handler(f): with master.handlecontext(): ret = f(master, message) if handling: - # Python2/3 compatibility hack - fn = getattr(f, "func_name", None) or getattr(f, "__name__") - master.addons(fn, message) + master.addons(f.__name__, message) if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() -- cgit v1.2.3 From a8a083a10d09509be8f39fb4463a857e0d3e2eeb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 10:08:41 +1200 Subject: Move StickyAuth into addon This is the first addon that handles an OptionsError, so this commit also demos how this works in console. Handling of command-line erorrs is on its way. --- mitmproxy/builtins/__init__.py | 2 ++ mitmproxy/builtins/stickyauth.py | 28 ++++++++++++++++++++++++ mitmproxy/console/master.py | 8 +++---- mitmproxy/console/options.py | 4 ++-- mitmproxy/console/statusbar.py | 4 ++-- mitmproxy/dump.py | 3 --- mitmproxy/flow/__init__.py | 5 ++--- mitmproxy/flow/master.py | 16 -------------- mitmproxy/flow/modules.py | 17 --------------- test/mitmproxy/builtins/test_stickyauth.py | 23 ++++++++++++++++++++ test/mitmproxy/test_flow.py | 34 ------------------------------ 11 files changed, 62 insertions(+), 82 deletions(-) create mode 100644 mitmproxy/builtins/stickyauth.py create mode 100644 test/mitmproxy/builtins/test_stickyauth.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index 59173145..867ebb22 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -1,9 +1,11 @@ from __future__ import absolute_import, print_function, division from mitmproxy.builtins import anticomp +from mitmproxy.builtins import stickyauth def default_addons(): return [ anticomp.AntiComp(), + stickyauth.StickyAuth(), ] diff --git a/mitmproxy/builtins/stickyauth.py b/mitmproxy/builtins/stickyauth.py new file mode 100644 index 00000000..1309911c --- /dev/null +++ b/mitmproxy/builtins/stickyauth.py @@ -0,0 +1,28 @@ +from __future__ import absolute_import, print_function, division + +from mitmproxy import filt +from mitmproxy import exceptions + + +class StickyAuth: + def __init__(self): + # Compiled filter + self.flt = None + self.hosts = {} + + def configure(self, options): + if options.stickyauth: + flt = filt.parse(options.stickyauth) + if not flt: + raise exceptions.OptionsError( + "stickyauth: invalid filter expression: %s" % options.stickyauth + ) + self.flt = flt + + def request(self, flow): + host = flow.request.host + if "authorization" in flow.request.headers: + self.hosts[host] = flow.request.headers["authorization"] + elif flow.match(self.flt): + if host in self.hosts: + flow.request.headers["authorization"] = self.hosts[host] diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 06a05464..8e5dae6b 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -244,11 +244,6 @@ class ConsoleMaster(flow.FlowMaster): print("Sticky cookies error: {}".format(r), file=sys.stderr) sys.exit(1) - r = self.set_stickyauth(options.stickyauth) - if r: - print("Sticky auth error: {}".format(r), file=sys.stderr) - sys.exit(1) - self.set_stream_large_bodies(options.stream_large_bodies) self.refresh_server_playback = options.refresh_server_playback @@ -300,6 +295,9 @@ class ConsoleMaster(flow.FlowMaster): self.__dict__[name] = value signals.update_settings.send(self) + def set_stickyauth(self, txt): + self.options.stickyauth = txt + def options_error(self, opts, exc): signals.status_message.send( message=str(exc), diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index db6d405a..331c28a5 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -120,7 +120,7 @@ class Options(urwid.WidgetWrap): select.Option( "Sticky Auth", "A", - lambda: master.stickyauth_txt, + lambda: master.options.stickyauth, self.sticky_auth ), select.Option( @@ -262,7 +262,7 @@ class Options(urwid.WidgetWrap): def sticky_auth(self): signals.status_prompt.send( prompt = "Sticky auth filter", - text = self.master.stickyauth_txt, + text = self.master.options.stickyauth, callback = self.master.set_stickyauth ) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index 543c6771..d0a24018 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -177,10 +177,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "t")) r.append(":%s]" % self.master.stickycookie_txt) - if self.master.stickyauth_txt: + if self.master.options.stickyauth: r.append("[") r.append(("heading_key", "u")) - r.append(":%s]" % self.master.stickyauth_txt) + r.append(":%s]" % self.master.options.stickyauth) if self.master.state.default_body_view.name != "Auto": r.append("[") r.append(("heading_key", "M")) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index cf7aa5c9..f69e3777 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -87,9 +87,6 @@ class DumpMaster(flow.FlowMaster): if options.stickycookie: self.set_stickycookie(options.stickycookie) - if options.stickyauth: - self.set_stickyauth(options.stickyauth) - if options.outfile: err = self.start_stream_to_path( options.outfile[0], diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index c14a0fec..2a5d9b38 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -5,7 +5,7 @@ from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_f from mitmproxy.flow.master import FlowMaster from mitmproxy.flow.modules import ( AppRegistry, ReplaceHooks, SetHeaders, StreamLargeBodies, ClientPlaybackState, - ServerPlaybackState, StickyCookieState, StickyAuthState + ServerPlaybackState, StickyCookieState ) from mitmproxy.flow.state import State, FlowView @@ -16,6 +16,5 @@ __all__ = [ "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", "FlowMaster", "AppRegistry", "ReplaceHooks", "SetHeaders", "StreamLargeBodies", "ClientPlaybackState", - "ServerPlaybackState", "StickyCookieState", "StickyAuthState", - "State", "FlowView", + "ServerPlaybackState", "StickyCookieState", "State", "FlowView", ] diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index ed2ee138..e469c499 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -42,9 +42,6 @@ class FlowMaster(controller.Master): self.stickycookie_state = None # type: Optional[modules.StickyCookieState] self.stickycookie_txt = None - self.stickyauth_state = False # type: Optional[modules.StickyAuthState] - self.stickyauth_txt = None - self.anticache = False self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] self.refresh_server_playback = False @@ -136,17 +133,6 @@ class FlowMaster(controller.Master): else: self.stream_large_bodies = False - def set_stickyauth(self, txt): - if txt: - flt = filt.parse(txt) - if not flt: - return "Invalid filter expression." - self.stickyauth_state = modules.StickyAuthState(flt) - self.stickyauth_txt = txt - else: - self.stickyauth_state = None - self.stickyauth_txt = None - def start_client_playback(self, flows, exit): """ flows: List of flows. @@ -326,8 +312,6 @@ class FlowMaster(controller.Master): def process_new_request(self, f): if self.stickycookie_state: self.stickycookie_state.handle_request(f) - if self.stickyauth_state: - self.stickyauth_state.handle_request(f) if self.anticache: f.request.anticache() diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index ab41da8d..83228a04 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -350,20 +350,3 @@ class StickyCookieState: if l: f.request.stickycookie = True f.request.headers["cookie"] = "; ".join(l) - - -class StickyAuthState: - def __init__(self, flt): - """ - flt: Compiled filter. - """ - self.flt = flt - self.hosts = {} - - def handle_request(self, f): - host = f.request.host - if "authorization" in f.request.headers: - self.hosts[host] = f.request.headers["authorization"] - elif f.match(self.flt): - if host in self.hosts: - f.request.headers["authorization"] = self.hosts[host] diff --git a/test/mitmproxy/builtins/test_stickyauth.py b/test/mitmproxy/builtins/test_stickyauth.py new file mode 100644 index 00000000..9233f435 --- /dev/null +++ b/test/mitmproxy/builtins/test_stickyauth.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickyauth +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options + + +class TestStickyAuth(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(stickyauth = ".*"), None, s) + sa = stickyauth.StickyAuth() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + f.request.headers["authorization"] = "foo" + self.invoke(m, "request", f) + + assert "address" in sa.hosts + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert f.request.headers["authorization"] == "foo" diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 6b270872..a6a3038c 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -120,20 +120,6 @@ class TestStickyCookieState: assert "cookie" in f.request.headers -class TestStickyAuthState: - - def test_response(self): - s = flow.StickyAuthState(filt.parse(".*")) - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - s.handle_request(f) - assert "address" in s.hosts - - f = tutils.tflow(resp=True) - s.handle_request(f) - assert f.request.headers["authorization"] == "foo" - - class TestClientPlaybackState: def test_tick(self): @@ -1004,26 +990,6 @@ class TestFlowMaster: fm.request(f) assert f.request.headers["cookie"] == "foo=bar" - def test_stickyauth(self): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - assert "Invalid" in fm.set_stickyauth("~h") - fm.set_stickyauth(".*") - assert fm.stickyauth_state - fm.set_stickyauth(None) - assert not fm.stickyauth_state - - fm.set_stickyauth(".*") - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - fm.request(f) - - f = tutils.tflow(resp=True) - assert fm.stickyauth_state.hosts - assert "authorization" not in f.request.headers - fm.request(f) - assert f.request.headers["authorization"] == "foo" - def test_stream(self): with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") -- cgit v1.2.3 From c52d567b4360ebbaa63918beb16a5f78649309a5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 10:19:16 +1200 Subject: Handle OptionsError for command-line args. --- mitmproxy/main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mitmproxy/main.py b/mitmproxy/main.py index bf01a3cb..5ced709b 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -76,7 +76,11 @@ def mitmproxy(args=None): # pragma: no cover server = get_server(console_options.no_server, proxy_config) - m = console.master.ConsoleMaster(server, console_options) + try: + m = console.master.ConsoleMaster(server, console_options) + except exceptions.OptionsError as e: + print("mitmproxy: %s" % e, file=sys.stderr) + sys.exit(1) try: m.run() except (KeyboardInterrupt, _thread.error): @@ -109,7 +113,7 @@ def mitmdump(args=None): # pragma: no cover signal.signal(signal.SIGTERM, cleankill) master.run() - except dump.DumpError as e: + except (dump.DumpError, exceptions.OptionsError) as e: print("mitmdump: %s" % e, file=sys.stderr) sys.exit(1) except (KeyboardInterrupt, _thread.error): @@ -137,7 +141,11 @@ def mitmweb(args=None): # pragma: no cover server = get_server(web_options.no_server, proxy_config) - m = web.master.WebMaster(server, web_options) + try: + m = web.master.WebMaster(server, web_options) + except exceptions.OptionsError as e: + print("mitmweb: %s" % e, file=sys.stderr) + sys.exit(1) try: m.run() except (KeyboardInterrupt, _thread.error): -- cgit v1.2.3 From 97b2e6d958a691060746b530219bf15a0bede1ae Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 11:11:41 +1200 Subject: Add a .setter helper to Options This returns a function that sets a named attribute, and is handy for event-driven code like mitmproxy console. --- mitmproxy/console/common.py | 2 +- mitmproxy/console/master.py | 3 --- mitmproxy/console/options.py | 8 +++++--- mitmproxy/options.py | 5 +++++ mitmproxy/web/app.py | 4 ++-- test/mitmproxy/test_options.py | 14 +++++++++++--- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index 470db88d..66962729 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -38,7 +38,7 @@ def is_keypress(k): """ Is this input event a keypress? """ - if isinstance(k, basestring): + if isinstance(k, six.string_types): return True diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 8e5dae6b..605b0e23 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -295,9 +295,6 @@ class ConsoleMaster(flow.FlowMaster): self.__dict__[name] = value signals.update_settings.send(self) - def set_stickyauth(self, txt): - self.options.stickyauth = txt - def options_error(self, opts, exc): signals.status_message.send( message=str(exc), diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index 331c28a5..f6342814 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -153,7 +153,6 @@ class Options(urwid.WidgetWrap): def clearall(self): self.master.anticache = False - self.master.anticomp = False self.master.killextra = False self.master.showhost = False self.master.refresh_server_playback = True @@ -163,8 +162,11 @@ class Options(urwid.WidgetWrap): self.master.set_ignore_filter([]) self.master.set_tcp_filter([]) self.master.scripts = [] - self.master.set_stickyauth(None) self.master.set_stickycookie(None) + + self.master.options.stickyauth = None + self.master.options.anticomp = False + self.master.state.default_body_view = contentviews.get("Auto") signals.update_settings.send(self) @@ -263,7 +265,7 @@ class Options(urwid.WidgetWrap): signals.status_prompt.send( prompt = "Sticky auth filter", text = self.master.options.stickyauth, - callback = self.master.set_stickyauth + callback = self.master.options.setter("stickyauth") ) def sticky_cookie(self): diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 0cc5fee1..5599185d 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -65,5 +65,10 @@ class Options(object): self._opts.update(kwargs) self.changed.send(self) + def setter(self, attr): + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + return lambda x: self.__setattr__(attr, x) + def __repr__(self): return pprint.pformat(self._opts) diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index f9d0dca6..ad149270 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -344,7 +344,7 @@ class Settings(RequestHandler): http2=self.master.server.config.http2, anticache=self.master.options.anticache, anticomp=self.master.options.anticomp, - stickyauth=self.master.stickyauth_txt, + stickyauth=self.master.options.stickyauth, stickycookie=self.master.stickycookie_txt, stream= self.master.stream_large_bodies.max_size if self.master.stream_large_bodies else False ) @@ -378,7 +378,7 @@ class Settings(RequestHandler): self.master.set_stickycookie(v) update[k] = v elif k == "stickyauth": - self.master.set_stickyauth(v) + self.master.options.stickyauth = v update[k] = v elif k == "stream": self.master.set_stream_large_bodies(v) diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py index 5fdb7abe..97db9430 100644 --- a/test/mitmproxy/test_options.py +++ b/test/mitmproxy/test_options.py @@ -3,7 +3,7 @@ import copy from mitmproxy import options from mitmproxy import exceptions -from netlib.tutils import raises +from netlib import tutils class TO(options.Options): @@ -19,8 +19,8 @@ def test_options(): assert o.two == "three" o.one = "one" assert o.one == "one" - raises("no such option", setattr, o, "nonexistent", "value") - raises("no such option", o.update, nonexistent = "value") + tutils.raises("no such option", setattr, o, "nonexistent", "value") + tutils.raises("no such option", o.update, nonexistent = "value") rec = [] @@ -38,6 +38,14 @@ def test_options(): assert rec[-1].one == "oink" +def test_setter(): + o = TO(two="three") + f = o.setter("two") + f("xxx") + assert o.two == "xxx" + tutils.raises("no such option", o.setter, "nonexistent") + + def test_rollback(): o = TO(one="two") -- cgit v1.2.3 From 6e7b86cd82dd9b379b0a57d423bbdcd74ba47256 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 11:12:37 +1200 Subject: Disable OSX tests on Travis for now We'll re-enable as soon as this is sorted out: https://github.com/travis-ci/travis-ci/issues/6303 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e9566ebe..e832d058 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,10 @@ matrix: include: - python: 3.5 env: TOXENV=lint - - os: osx - osx_image: xcode7.3 - language: generic - env: TOXENV=py35 +# - os: osx +# osx_image: xcode7.3 +# language: generic +# env: TOXENV=py35 - python: 3.5 env: TOXENV=py35 - python: 3.5 -- cgit v1.2.3 From 143bf0dfa9138a4340287d636bb68648665b3829 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 11:24:06 +1200 Subject: AntiCache to addon --- mitmproxy/builtins/__init__.py | 2 ++ mitmproxy/builtins/anticache.py | 13 +++++++++++++ mitmproxy/console/options.py | 8 ++++---- mitmproxy/console/statusbar.py | 2 +- mitmproxy/dump.py | 2 -- mitmproxy/flow/master.py | 4 ---- test/mitmproxy/builtins/test_anticache.py | 23 +++++++++++++++++++++++ test/mitmproxy/test_flow.py | 10 ---------- 8 files changed, 43 insertions(+), 21 deletions(-) create mode 100644 mitmproxy/builtins/anticache.py create mode 100644 test/mitmproxy/builtins/test_anticache.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index 867ebb22..b5419378 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -1,11 +1,13 @@ from __future__ import absolute_import, print_function, division +from mitmproxy.builtins import anticache from mitmproxy.builtins import anticomp from mitmproxy.builtins import stickyauth def default_addons(): return [ + anticache.AntiCache(), anticomp.AntiComp(), stickyauth.StickyAuth(), ] diff --git a/mitmproxy/builtins/anticache.py b/mitmproxy/builtins/anticache.py new file mode 100644 index 00000000..f208e2fb --- /dev/null +++ b/mitmproxy/builtins/anticache.py @@ -0,0 +1,13 @@ +from __future__ import absolute_import, print_function, division + + +class AntiCache: + def __init__(self): + self.enabled = False + + def configure(self, options): + self.enabled = options.anticache + + def request(self, flow): + if self.enabled: + flow.request.anticache() diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index f6342814..c76a058f 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -96,7 +96,7 @@ class Options(urwid.WidgetWrap): select.Option( "Anti-Cache", "a", - lambda: master.anticache, + lambda: master.options.anticache, self.toggle_anticache ), select.Option( @@ -152,7 +152,6 @@ class Options(urwid.WidgetWrap): return super(self.__class__, self).keypress(size, key) def clearall(self): - self.master.anticache = False self.master.killextra = False self.master.showhost = False self.master.refresh_server_playback = True @@ -164,8 +163,9 @@ class Options(urwid.WidgetWrap): self.master.scripts = [] self.master.set_stickycookie(None) - self.master.options.stickyauth = None + self.master.options.anticache = False self.master.options.anticomp = False + self.master.options.stickyauth = None self.master.state.default_body_view = contentviews.get("Auto") @@ -176,7 +176,7 @@ class Options(urwid.WidgetWrap): ) def toggle_anticache(self): - self.master.anticache = not self.master.anticache + self.master.options.anticache = not self.master.options.anticache def toggle_anticomp(self): self.master.options.anticomp = not self.master.options.anticomp diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index d0a24018..1357d7ca 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -187,7 +187,7 @@ class StatusBar(urwid.WidgetWrap): r.append(":%s]" % self.master.state.default_body_view.name) opts = [] - if self.master.anticache: + if self.master.options.anticache: opts.append("anticache") if self.master.options.anticomp: opts.append("anticomp") diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index f69e3777..b953d131 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -63,8 +63,6 @@ class DumpMaster(flow.FlowMaster): self.addons.add(*builtins.default_addons()) self.outfile = outfile self.o = options - self.anticache = options.anticache - self.anticomp = options.anticomp self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index e469c499..06e1b460 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -42,7 +42,6 @@ class FlowMaster(controller.Master): self.stickycookie_state = None # type: Optional[modules.StickyCookieState] self.stickycookie_txt = None - self.anticache = False self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] self.refresh_server_playback = False self.replacehooks = modules.ReplaceHooks() @@ -313,9 +312,6 @@ class FlowMaster(controller.Master): if self.stickycookie_state: self.stickycookie_state.handle_request(f) - if self.anticache: - f.request.anticache() - if self.server_playback: pb = self.do_server_playback(f) if not pb and self.kill_nonreplay: diff --git a/test/mitmproxy/builtins/test_anticache.py b/test/mitmproxy/builtins/test_anticache.py new file mode 100644 index 00000000..5a00af03 --- /dev/null +++ b/test/mitmproxy/builtins/test_anticache.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticache +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options + + +class TestAntiCache(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticache = True), None, s) + sa = anticache.AntiCache() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + f.request.headers["if-modified-since"] = "test" + f.request.headers["if-none-match"] = "test" + self.invoke(m, "request", f) + assert "if-modified-since" not in f.request.headers + assert "if-none-match" not in f.request.headers diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index a6a3038c..585dbf93 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -855,7 +855,6 @@ class TestFlowMaster: def test_all(self): s = flow.State() fm = flow.FlowMaster(None, None, s) - fm.anticache = True f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) f.request = HTTPRequest.wrap(netlib.tutils.treq()) @@ -1053,15 +1052,6 @@ class TestRequest: assert r.url == "https://address:22/path" assert r.pretty_url == "https://foo.com:22/path" - def test_anticache(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) - r.headers = Headers() - r.headers["if-modified-since"] = "test" - r.headers["if-none-match"] = "test" - r.anticache() - assert "if-modified-since" not in r.headers - assert "if-none-match" not in r.headers - def test_replace(self): r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" -- cgit v1.2.3 From 00509d86a8ee3057ab870ae54dd1baf3ea10946b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 12:10:46 +1200 Subject: StickyCookies to addon --- mitmproxy/builtins/__init__.py | 2 + mitmproxy/builtins/stickycookie.py | 70 ++++++++++++++++ mitmproxy/console/master.py | 5 -- mitmproxy/console/options.py | 8 +- mitmproxy/console/statusbar.py | 4 +- mitmproxy/dump.py | 7 +- mitmproxy/flow/master.py | 23 ------ test/mitmproxy/builtins/test_stickycookie.py | 118 +++++++++++++++++++++++++++ test/mitmproxy/test_flow.py | 102 ----------------------- 9 files changed, 199 insertions(+), 140 deletions(-) create mode 100644 mitmproxy/builtins/stickycookie.py create mode 100644 test/mitmproxy/builtins/test_stickycookie.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index b5419378..75326712 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, print_function, division from mitmproxy.builtins import anticache from mitmproxy.builtins import anticomp from mitmproxy.builtins import stickyauth +from mitmproxy.builtins import stickycookie def default_addons(): @@ -10,4 +11,5 @@ def default_addons(): anticache.AntiCache(), anticomp.AntiComp(), stickyauth.StickyAuth(), + stickycookie.StickyCookie(), ] diff --git a/mitmproxy/builtins/stickycookie.py b/mitmproxy/builtins/stickycookie.py new file mode 100644 index 00000000..5913976c --- /dev/null +++ b/mitmproxy/builtins/stickycookie.py @@ -0,0 +1,70 @@ +import collections +from six.moves import http_cookiejar +from netlib.http import cookies + +from mitmproxy import exceptions +from mitmproxy import filt + + +def ckey(attrs, f): + """ + Returns a (domain, port, path) tuple. + """ + domain = f.request.host + path = "/" + if "domain" in attrs: + domain = attrs["domain"] + if "path" in attrs: + path = attrs["path"] + return (domain, f.request.port, path) + + +def domain_match(a, b): + if http_cookiejar.domain_match(a, b): + return True + elif http_cookiejar.domain_match(a, b.strip(".")): + return True + return False + + +class StickyCookie: + def __init__(self): + self.jar = collections.defaultdict(dict) + self.flt = None + + def configure(self, options): + if options.stickycookie: + flt = filt.parse(options.stickycookie) + if not flt: + raise exceptions.OptionsError( + "stickycookie: invalid filter expression: %s" % options.stickycookie + ) + self.flt = flt + + def response(self, flow): + if self.flt: + for name, (value, attrs) in flow.response.cookies.items(multi=True): + # FIXME: We now know that Cookie.py screws up some cookies with + # valid RFC 822/1123 datetime specifications for expiry. Sigh. + a = ckey(attrs, flow) + if domain_match(flow.request.host, a[0]): + b = attrs.with_insert(0, name, value) + self.jar[a][name] = b + + def request(self, flow): + if self.flt: + l = [] + if flow.match(self.flt): + for domain, port, path in self.jar.keys(): + match = [ + domain_match(flow.request.host, domain), + flow.request.port == port, + flow.request.path.startswith(path) + ] + if all(match): + c = self.jar[(domain, port, path)] + l.extend([cookies.format_cookie_header(c[name].items(multi=True)) for name in c.keys()]) + if l: + # FIXME: we need to formalise this... + flow.request.stickycookie = True + flow.request.headers["cookie"] = "; ".join(l) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 605b0e23..5c015033 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -239,11 +239,6 @@ class ConsoleMaster(flow.FlowMaster): if options.limit: self.set_limit(options.limit) - r = self.set_stickycookie(options.stickycookie) - if r: - print("Sticky cookies error: {}".format(r), file=sys.stderr) - sys.exit(1) - self.set_stream_large_bodies(options.stream_large_bodies) self.refresh_server_playback = options.refresh_server_playback diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index c76a058f..d363ba74 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -126,7 +126,7 @@ class Options(urwid.WidgetWrap): select.Option( "Sticky Cookies", "t", - lambda: master.stickycookie_txt, + lambda: master.options.stickycookie, self.sticky_cookie ), ] @@ -161,11 +161,11 @@ class Options(urwid.WidgetWrap): self.master.set_ignore_filter([]) self.master.set_tcp_filter([]) self.master.scripts = [] - self.master.set_stickycookie(None) self.master.options.anticache = False self.master.options.anticomp = False self.master.options.stickyauth = None + self.master.options.stickycookie = None self.master.state.default_body_view = contentviews.get("Auto") @@ -271,8 +271,8 @@ class Options(urwid.WidgetWrap): def sticky_cookie(self): signals.status_prompt.send( prompt = "Sticky cookie filter", - text = self.master.stickycookie_txt, - callback = self.master.set_stickycookie + text = self.master.options.stickycookie, + callback = self.master.options.setter("stickycookie") ) def palette(self): diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index 1357d7ca..fc41869c 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -173,10 +173,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "Marked Flows")) r.append("]") - if self.master.stickycookie_txt: + if self.master.options.stickycookie: r.append("[") r.append(("heading_key", "t")) - r.append(":%s]" % self.master.stickycookie_txt) + r.append(":%s]" % self.master.options.stickycookie) if self.master.options.stickyauth: r.append("[") r.append(("heading_key", "u")) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index b953d131..d9142a4d 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -82,9 +82,6 @@ class DumpMaster(flow.FlowMaster): else: self.filt = None - if options.stickycookie: - self.set_stickycookie(options.stickycookie) - if options.outfile: err = self.start_stream_to_path( options.outfile[0], @@ -230,7 +227,9 @@ class DumpMaster(flow.FlowMaster): def _echo_request_line(self, flow): if flow.request.stickycookie: - stickycookie = click.style("[stickycookie] ", fg="yellow", bold=True) + stickycookie = click.style( + "[stickycookie] ", fg="yellow", bold=True + ) else: stickycookie = "" diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 06e1b460..d67ee7cc 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -8,7 +8,6 @@ from typing import List, Optional, Set # noqa import netlib.exceptions from mitmproxy import controller from mitmproxy import exceptions -from mitmproxy import filt from mitmproxy import models from mitmproxy import script from mitmproxy.flow import io @@ -39,9 +38,6 @@ class FlowMaster(controller.Master): self.scripts = [] # type: List[script.Script] self.pause_scripts = False - self.stickycookie_state = None # type: Optional[modules.StickyCookieState] - self.stickycookie_txt = None - self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] self.refresh_server_playback = False self.replacehooks = modules.ReplaceHooks() @@ -115,17 +111,6 @@ class FlowMaster(controller.Master): def set_tcp_filter(self, host_patterns): self.server.config.check_tcp = HostMatcher(host_patterns) - def set_stickycookie(self, txt): - if txt: - flt = filt.parse(txt) - if not flt: - return "Invalid filter expression." - self.stickycookie_state = modules.StickyCookieState(flt) - self.stickycookie_txt = txt - else: - self.stickycookie_state = None - self.stickycookie_txt = None - def set_stream_large_bodies(self, max_size): if max_size is not None: self.stream_large_bodies = modules.StreamLargeBodies(max_size) @@ -309,18 +294,11 @@ class FlowMaster(controller.Master): raise exceptions.FlowReadException(v.strerror) def process_new_request(self, f): - if self.stickycookie_state: - self.stickycookie_state.handle_request(f) - if self.server_playback: pb = self.do_server_playback(f) if not pb and self.kill_nonreplay: f.kill(self) - def process_new_response(self, f): - if self.stickycookie_state: - self.stickycookie_state.handle_response(f) - def replay_request(self, f, block=False, run_scripthooks=True): """ Returns None if successful, or error message if not. @@ -431,7 +409,6 @@ class FlowMaster(controller.Master): if not f.reply.acked: if self.client_playback: self.client_playback.clear(f) - self.process_new_response(f) if self.stream: self.stream.add(f) return f diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py new file mode 100644 index 00000000..ddfaf5e0 --- /dev/null +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -0,0 +1,118 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickycookie +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options +from netlib import tutils as ntutils + + +def test_domain_match(): + assert stickycookie.domain_match("www.google.com", ".google.com") + assert stickycookie.domain_match("google.com", ".google.com") + + +class TestStickyCookie(mastertest.MasterTest): + def mk(self): + s = state.State() + m = master.FlowMaster(options.Options(stickycookie = ".*"), None, s) + sc = stickycookie.StickyCookie() + m.addons.add(sc) + return s, m, sc + + def test_config(self): + sc = stickycookie.StickyCookie() + tutils.raises( + "invalid filter", + sc.configure, + options.Options(stickycookie = "~b") + ) + + def test_simple(self): + s, m, sc = self.mk() + m.addons.add(sc) + + f = tutils.tflow(resp=True) + f.response.headers["set-cookie"] = "foo=bar" + self.invoke(m, "request", f) + + f.reply.acked = False + self.invoke(m, "response", f) + + assert sc.jar + assert "cookie" not in f.request.headers + + f = f.copy() + f.reply.acked = False + self.invoke(m, "request", f) + assert f.request.headers["cookie"] == "foo=bar" + + def _response(self, s, m, sc, cookie, host): + f = tutils.tflow(req=ntutils.treq(host=host, port=80), resp=True) + f.response.headers["Set-Cookie"] = cookie + self.invoke(m, "response", f) + return f + + def test_response(self): + s, m, sc = self.mk() + + c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ + "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " + + self._response(s, m, sc, c, "host") + assert not sc.jar.keys() + + self._response(s, m, sc, c, "www.google.com") + assert sc.jar.keys() + + self._response(s, m, sc, "SSID=mooo", "www.google.com") + assert sc.jar.keys()[0] == ('www.google.com', 80, '/') + + def test_response_multiple(self): + s, m, sc = self.mk() + + # Test setting of multiple cookies + c1 = "somecookie=test; Path=/" + c2 = "othercookie=helloworld; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = sc.jar.keys()[0] + assert len(sc.jar[googlekey].keys()) == 2 + + def test_response_weird(self): + s, m, sc = self.mk() + + # Test setting of weird cookie keys + f = tutils.tflow(req=ntutils.treq(host="www.google.com", port=80), resp=True) + cs = [ + "foo/bar=hello", + "foo:bar=world", + "foo@bar=fizz", + "foo,bar=buzz", + ] + for c in cs: + f.response.headers["Set-Cookie"] = c + self.invoke(m, "response", f) + googlekey = sc.jar.keys()[0] + assert len(sc.jar[googlekey].keys()) == len(cs) + + def test_response_overwrite(self): + s, m, sc = self.mk() + + # Test overwriting of a cookie value + c1 = "somecookie=helloworld; Path=/" + c2 = "somecookie=newvalue; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = sc.jar.keys()[0] + assert len(sc.jar[googlekey].keys()) == 1 + assert sc.jar[googlekey]["somecookie"].items()[0][1] == "newvalue" + + def test_request(self): + s, m, sc = self.mk() + + f = self._response(s, m, sc, "SSID=mooo", "www.google.com") + assert "cookie" not in f.request.headers + self.invoke(m, "request", f) + assert "cookie" in f.request.headers diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 585dbf93..91342e58 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -40,86 +40,6 @@ def test_app_registry(): assert ar.get(r) -class TestStickyCookieState: - - def _response(self, cookie, host): - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True) - f.response.headers["Set-Cookie"] = cookie - s.handle_response(f) - return s, f - - def test_domain_match(self): - s = flow.StickyCookieState(filt.parse(".*")) - assert s.domain_match("www.google.com", ".google.com") - assert s.domain_match("google.com", ".google.com") - - def test_response(self): - c = ( - "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " - "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " - ) - - s, f = self._response(c, "host") - assert not s.jar.keys() - - s, f = self._response(c, "www.google.com") - assert list(s.jar.keys())[0] == ('.google.com', 80, '/') - - s, f = self._response("SSID=mooo", "www.google.com") - assert list(s.jar.keys())[0] == ('www.google.com', 80, '/') - - # Test setting of multiple cookies - c1 = "somecookie=test; Path=/" - c2 = "othercookie=helloworld; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey].keys()) == 2 - - # Test setting of weird cookie keys - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True) - cs = [ - "foo/bar=hello", - "foo:bar=world", - "foo@bar=fizz", - "foo,bar=buzz", - ] - for c in cs: - f.response.headers["Set-Cookie"] = c - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey]) == len(cs) - - # Test overwriting of a cookie value - c1 = "somecookie=helloworld; Path=/" - c2 = "somecookie=newvalue; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey]) == 1 - assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue" - - def test_response_delete(self): - c = "duffer=zafar; Path=/", "www.google.com" - - # Test that a cookie is be deleted - # by setting the expire time in the past - s, f = self._response(*c) - f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT" - s.handle_response(f) - assert not s.jar.keys() - - def test_request(self): - s, f = self._response("SSID=mooo", b"www.google.com") - assert "cookie" not in f.request.headers - s.handle_request(f) - assert "cookie" in f.request.headers - - class TestClientPlaybackState: def test_tick(self): @@ -967,28 +887,6 @@ class TestFlowMaster: fm.process_new_request(f) assert "killed" in f.error.msg - def test_stickycookie(self): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - assert "Invalid" in fm.set_stickycookie("~h") - fm.set_stickycookie(".*") - assert fm.stickycookie_state - fm.set_stickycookie(None) - assert not fm.stickycookie_state - - fm.set_stickycookie(".*") - f = tutils.tflow(resp=True) - f.response.headers["set-cookie"] = "foo=bar" - fm.request(f) - f.reply.acked = False - fm.response(f) - assert fm.stickycookie_state.jar - assert "cookie" not in f.request.headers - f = f.copy() - f.reply.acked = False - fm.request(f) - assert f.request.headers["cookie"] == "foo=bar" - def test_stream(self): with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") -- cgit v1.2.3 From 120465a142434af224f4f3219d199af8bfb0619c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 12:17:28 +1200 Subject: Ditch StickyCookie module --- mitmproxy/flow/__init__.py | 4 +-- mitmproxy/flow/modules.py | 63 ---------------------------------------------- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index 2a5d9b38..4c3bb828 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -5,7 +5,7 @@ from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_f from mitmproxy.flow.master import FlowMaster from mitmproxy.flow.modules import ( AppRegistry, ReplaceHooks, SetHeaders, StreamLargeBodies, ClientPlaybackState, - ServerPlaybackState, StickyCookieState + ServerPlaybackState ) from mitmproxy.flow.state import State, FlowView @@ -16,5 +16,5 @@ __all__ = [ "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", "FlowMaster", "AppRegistry", "ReplaceHooks", "SetHeaders", "StreamLargeBodies", "ClientPlaybackState", - "ServerPlaybackState", "StickyCookieState", "State", "FlowView", + "ServerPlaybackState", "State", "FlowView", ] diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index 83228a04..84c11ac2 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -287,66 +287,3 @@ class ServerPlaybackState: return l[0] else: return l.pop(0) - - -class StickyCookieState: - def __init__(self, flt): - """ - flt: Compiled filter. - """ - self.jar = collections.defaultdict(dict) - self.flt = flt - - def ckey(self, attrs, f): - """ - Returns a (domain, port, path) tuple. - """ - domain = f.request.host - path = "/" - if "domain" in attrs: - domain = attrs["domain"] - if "path" in attrs: - path = attrs["path"] - return (domain, f.request.port, path) - - def domain_match(self, a, b): - if http_cookiejar.domain_match(a, b): - return True - elif http_cookiejar.domain_match(a, b.strip(".")): - return True - return False - - def handle_response(self, f): - for name, (value, attrs) in f.response.cookies.items(multi=True): - # FIXME: We now know that Cookie.py screws up some cookies with - # valid RFC 822/1123 datetime specifications for expiry. Sigh. - dom_port_path = self.ckey(attrs, f) - - if self.domain_match(f.request.host, dom_port_path[0]): - if cookies.is_expired(attrs): - # Remove the cookie from jar - self.jar[dom_port_path].pop(name, None) - - # If all cookies of a dom_port_path have been removed - # then remove it from the jar itself - if not self.jar[dom_port_path]: - self.jar.pop(dom_port_path, None) - else: - b = attrs.with_insert(0, name, value) - self.jar[dom_port_path][name] = b - - def handle_request(self, f): - l = [] - if f.match(self.flt): - for domain, port, path in self.jar.keys(): - match = [ - self.domain_match(f.request.host, domain), - f.request.port == port, - f.request.path.startswith(path) - ] - if all(match): - c = self.jar[(domain, port, path)] - l.extend([cookies.format_cookie_header(c[name].items(multi=True)) for name in c.keys()]) - if l: - f.request.stickycookie = True - f.request.headers["cookie"] = "; ".join(l) -- cgit v1.2.3 From cf3b3d206a22d47b4b5cc5ef57c09c7c577b27aa Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 12:45:00 +1200 Subject: Zap unused imports --- mitmproxy/flow/modules.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index 84c11ac2..2ad514f0 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,10 +1,8 @@ from __future__ import absolute_import, print_function, division -import collections import hashlib import re -from six.moves import http_cookiejar from six.moves import urllib from mitmproxy import controller @@ -12,7 +10,6 @@ from mitmproxy import filt from netlib import wsgi from netlib import version from netlib import strutils -from netlib.http import cookies from netlib.http import http1 -- cgit v1.2.3 From 703c05066ec0bc05c680e24d368606791dd1c958 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 12:59:00 +1200 Subject: Fix indeterminacy in sticky cookie tests How has this not bitten us before? --- test/mitmproxy/builtins/test_stickycookie.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py index ddfaf5e0..e64ecb5b 100644 --- a/test/mitmproxy/builtins/test_stickycookie.py +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -64,8 +64,11 @@ class TestStickyCookie(mastertest.MasterTest): self._response(s, m, sc, c, "www.google.com") assert sc.jar.keys() - self._response(s, m, sc, "SSID=mooo", "www.google.com") - assert sc.jar.keys()[0] == ('www.google.com', 80, '/') + sc.jar.clear() + self._response( + s, m, sc, "SSID=mooo", "www.google.com" + ) + assert list(sc.jar.keys())[0] == ('www.google.com', 80, '/') def test_response_multiple(self): s, m, sc = self.mk() @@ -76,7 +79,7 @@ class TestStickyCookie(mastertest.MasterTest): f = self._response(s, m, sc, c1, "www.google.com") f.response.headers["Set-Cookie"] = c2 self.invoke(m, "response", f) - googlekey = sc.jar.keys()[0] + googlekey = list(sc.jar.keys())[0] assert len(sc.jar[googlekey].keys()) == 2 def test_response_weird(self): @@ -93,7 +96,7 @@ class TestStickyCookie(mastertest.MasterTest): for c in cs: f.response.headers["Set-Cookie"] = c self.invoke(m, "response", f) - googlekey = sc.jar.keys()[0] + googlekey = list(sc.jar.keys())[0] assert len(sc.jar[googlekey].keys()) == len(cs) def test_response_overwrite(self): @@ -105,9 +108,9 @@ class TestStickyCookie(mastertest.MasterTest): f = self._response(s, m, sc, c1, "www.google.com") f.response.headers["Set-Cookie"] = c2 self.invoke(m, "response", f) - googlekey = sc.jar.keys()[0] + googlekey = list(sc.jar.keys())[0] assert len(sc.jar[googlekey].keys()) == 1 - assert sc.jar[googlekey]["somecookie"].items()[0][1] == "newvalue" + assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" def test_request(self): s, m, sc = self.mk() -- cgit v1.2.3 From b2c4f301cb834ecdf6e5b0063e86be877d3ece6d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 12:43:17 +1200 Subject: Stream to file -> addon This commit also clarifies a confusion about the "outfile" attribute and its use in testing in the mitmdump master. --- mitmproxy/builtins/__init__.py | 2 + mitmproxy/builtins/stream.py | 54 +++++++++++++++++++++ mitmproxy/dump.py | 22 +++------ mitmproxy/flow/master.py | 28 ----------- test/mitmproxy/builtins/test_stream.py | 46 ++++++++++++++++++ test/mitmproxy/mastertest.py | 13 +++-- test/mitmproxy/test_dump.py | 89 ++++++++++++++++++++-------------- test/mitmproxy/test_flow.py | 31 ------------ 8 files changed, 167 insertions(+), 118 deletions(-) create mode 100644 mitmproxy/builtins/stream.py create mode 100644 test/mitmproxy/builtins/test_stream.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index 75326712..8021c20f 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -4,6 +4,7 @@ from mitmproxy.builtins import anticache from mitmproxy.builtins import anticomp from mitmproxy.builtins import stickyauth from mitmproxy.builtins import stickycookie +from mitmproxy.builtins import stream def default_addons(): @@ -12,4 +13,5 @@ def default_addons(): anticomp.AntiComp(), stickyauth.StickyAuth(), stickycookie.StickyCookie(), + stream.Stream(), ] diff --git a/mitmproxy/builtins/stream.py b/mitmproxy/builtins/stream.py new file mode 100644 index 00000000..821a71f1 --- /dev/null +++ b/mitmproxy/builtins/stream.py @@ -0,0 +1,54 @@ +from __future__ import absolute_import, print_function, division +import os.path + +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy.flow import io + + +class Stream: + def __init__(self): + self.stream = None + + def start_stream_to_path(self, path, mode, filt): + path = os.path.expanduser(path) + try: + f = open(path, mode) + except IOError as v: + return str(v) + self.stream = io.FilteredFlowWriter(f, filt) + + def configure(self, options): + # We're already streaming - stop the previous stream and restart + if self.stream: + self.done() + + if options.outfile: + filt = None + if options.get("filtstr"): + filt = filt.parse(options.filtstr) + if not filt: + raise exceptions.OptionsError( + "Invalid filter specification: %s" % options.filtstr + ) + path, mode = options.outfile + if mode not in ("wb", "ab"): + raise exceptions.OptionsError("Invalid mode.") + err = self.start_stream_to_path(path, mode, filt) + if err: + raise exceptions.OptionsError(err) + + def done(self): + if self.stream: + for flow in ctx.master.active_flows: + self.stream.add(flow) + self.stream.fo.close() + self.stream = None + + def tcp_close(self, flow): + if self.stream: + self.stream.add(flow) + + def response(self, flow): + if self.stream: + self.stream.add(flow) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index d9142a4d..296419db 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -52,16 +52,17 @@ class Options(options.Options): "replay_ignore_content", "replay_ignore_params", "replay_ignore_payload_params", - "replay_ignore_host" + "replay_ignore_host", + + "tfile" ] class DumpMaster(flow.FlowMaster): - def __init__(self, server, options, outfile=None): + def __init__(self, server, options): flow.FlowMaster.__init__(self, options, server, flow.State()) self.addons.add(*builtins.default_addons()) - self.outfile = outfile self.o = options self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params @@ -82,15 +83,6 @@ class DumpMaster(flow.FlowMaster): else: self.filt = None - if options.outfile: - err = self.start_stream_to_path( - options.outfile[0], - options.outfile[1], - self.filt - ) - if err: - raise DumpError(err) - if options.replacements: for i in options.replacements: self.replacehooks.add(*i) @@ -163,7 +155,7 @@ class DumpMaster(flow.FlowMaster): def echo(self, text, indent=None, **style): if indent: text = self.indent(indent, text) - click.secho(text, file=self.outfile, **style) + click.secho(text, file=self.options.tfile, **style) def _echo_message(self, message): if self.options.flow_detail >= 2 and hasattr(message, "headers"): @@ -312,8 +304,8 @@ class DumpMaster(flow.FlowMaster): if f.error: self.echo(" << {}".format(f.error.msg), bold=True, fg="red") - if self.outfile: - self.outfile.flush() + if self.options.tfile: + self.options.tfile.flush() def _process_flow(self, f): if self.filt and not f.match(self.filt): diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index d67ee7cc..27ceee87 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -46,7 +46,6 @@ class FlowMaster(controller.Master): self.replay_ignore_content = None self.replay_ignore_host = False - self.stream = None self.apps = modules.AppRegistry() def start_app(self, host, port): @@ -409,8 +408,6 @@ class FlowMaster(controller.Master): if not f.reply.acked: if self.client_playback: self.client_playback.clear(f) - if self.stream: - self.stream.add(f) return f def handle_intercept(self, f): @@ -471,33 +468,8 @@ class FlowMaster(controller.Master): @controller.handler def tcp_close(self, flow): self.active_flows.discard(flow) - if self.stream: - self.stream.add(flow) self.run_scripts("tcp_close", flow) def shutdown(self): super(FlowMaster, self).shutdown() - - # Add all flows that are still active - if self.stream: - for flow in self.active_flows: - self.stream.add(flow) - self.stop_stream() - self.unload_scripts() - - def start_stream(self, fp, filt): - self.stream = io.FilteredFlowWriter(fp, filt) - - def stop_stream(self): - self.stream.fo.close() - self.stream = None - - def start_stream_to_path(self, path, mode="wb", filt=None): - path = os.path.expanduser(path) - try: - f = open(path, mode) - self.start_stream(f, filt) - except IOError as v: - return str(v) - self.stream_path = path diff --git a/test/mitmproxy/builtins/test_stream.py b/test/mitmproxy/builtins/test_stream.py new file mode 100644 index 00000000..54e4f7d9 --- /dev/null +++ b/test/mitmproxy/builtins/test_stream.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import, print_function, division + +from .. import tutils, mastertest + +import os.path + +from mitmproxy.builtins import stream +from mitmproxy.flow import master, FlowReader +from mitmproxy.flow import state +from mitmproxy import options + + +class TestStream(mastertest.MasterTest): + def test_stream(self): + with tutils.tmpdir() as tdir: + p = os.path.join(tdir, "foo") + + def r(): + r = FlowReader(open(p, "rb")) + return list(r.stream()) + + s = state.State() + m = master.FlowMaster( + options.Options( + outfile = (p, "wb") + ), + None, + s + ) + sa = stream.Stream() + + m.addons.add(sa) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + self.invoke(m, "response", f) + m.addons.remove(sa) + + assert r()[0].response + + m.options.outfile = (p, "ab") + + m.addons.add(sa) + f = tutils.tflow() + self.invoke(m, "request", f) + m.addons.remove(sa) + assert not r()[1].response diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 06854e25..9754d3a9 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -18,15 +18,14 @@ class MasterTest: l = proxy.Log("connect") l.reply = mock.MagicMock() master.log(l) - master.clientconnect(f.client_conn) - master.serverconnect(f.server_conn) - master.request(f) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "serverconnect", f.server_conn) + self.invoke(master, "request", f) if not f.error: f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content)) - f.reply.acked = False - f = master.response(f) - f.client_conn.reply.acked = False - master.clientdisconnect(f.client_conn) + self.invoke(master, "response", f) + self.invoke(master, "clientdisconnect", f) return f def dummy_cycle(self, master, n, content): diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index aa73b5a4..9686be84 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -4,31 +4,33 @@ from mitmproxy.exceptions import ContentViewException import netlib.tutils -from mitmproxy import dump, flow, models +from mitmproxy import dump, flow, models, exceptions from . import tutils, mastertest import mock def test_strfuncs(): - o = dump.Options() + o = dump.Options( + tfile = StringIO(), + flow_detail = 0, + ) m = dump.DumpMaster(None, o) - m.outfile = StringIO() m.o.flow_detail = 0 m.echo_flow(tutils.tflow()) - assert not m.outfile.getvalue() + assert not o.tfile.getvalue() m.o.flow_detail = 4 m.echo_flow(tutils.tflow()) - assert m.outfile.getvalue() + assert o.tfile.getvalue() - m.outfile = StringIO() + o.tfile = StringIO() m.echo_flow(tutils.tflow(resp=True)) - assert "<<" in m.outfile.getvalue() + assert "<<" in o.tfile.getvalue() - m.outfile = StringIO() + o.tfile = StringIO() m.echo_flow(tutils.tflow(err=True)) - assert "<<" in m.outfile.getvalue() + assert "<<" in o.tfile.getvalue() flow = tutils.tflow() flow.request = netlib.tutils.treq() @@ -50,25 +52,32 @@ def test_strfuncs(): def test_contentview(get_content_view): get_content_view.side_effect = ContentViewException(""), ("x", iter([])) - o = dump.Options(flow_detail=4, verbosity=3) - m = dump.DumpMaster(None, o, StringIO()) + o = dump.Options( + flow_detail=4, + verbosity=3, + tfile=StringIO(), + ) + m = dump.DumpMaster(None, o) m.echo_flow(tutils.tflow()) - assert "Content viewer failed" in m.outfile.getvalue() + assert "Content viewer failed" in m.options.tfile.getvalue() class TestDumpMaster(mastertest.MasterTest): def dummy_cycle(self, master, n, content): mastertest.MasterTest.dummy_cycle(self, master, n, content) - return master.outfile.getvalue() + return master.options.tfile.getvalue() def mkmaster(self, filt, **options): - cs = StringIO() if "verbosity" not in options: options["verbosity"] = 0 if "flow_detail" not in options: options["flow_detail"] = 0 - o = dump.Options(filtstr=filt, **options) - return dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + filtstr=filt, + tfile=StringIO(), + **options + ) + return dump.DumpMaster(None, o) def test_basic(self): for i in (1, 2, 3): @@ -89,31 +98,33 @@ class TestDumpMaster(mastertest.MasterTest): ) def test_error(self): - cs = StringIO() - o = dump.Options(flow_detail=1) - m = dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + tfile=StringIO(), + flow_detail=1 + ) + m = dump.DumpMaster(None, o) f = tutils.tflow(err=True) m.request(f) assert m.error(f) - assert "error" in cs.getvalue() + assert "error" in o.tfile.getvalue() def test_missing_content(self): - cs = StringIO() - o = dump.Options(flow_detail=3) - m = dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + flow_detail=3, + tfile=StringIO(), + ) + m = dump.DumpMaster(None, o) f = tutils.tflow() f.request.content = None m.request(f) f.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) f.response.content = None m.response(f) - assert "content missing" in cs.getvalue() + assert "content missing" in o.tfile.getvalue() def test_replay(self): - cs = StringIO() - o = dump.Options(server_replay=["nonexistent"], kill=True) - tutils.raises(dump.DumpError, dump.DumpMaster, None, o, outfile=cs) + tutils.raises(dump.DumpError, dump.DumpMaster, None, o) with tutils.tmpdir() as t: p = os.path.join(t, "rep") @@ -122,7 +133,7 @@ class TestDumpMaster(mastertest.MasterTest): o = dump.Options(server_replay=[p], kill=True) o.verbosity = 0 o.flow_detail = 0 - m = dump.DumpMaster(None, o, outfile=cs) + m = dump.DumpMaster(None, o) self.cycle(m, b"content") self.cycle(m, b"content") @@ -130,13 +141,13 @@ class TestDumpMaster(mastertest.MasterTest): o = dump.Options(server_replay=[p], kill=False) o.verbosity = 0 o.flow_detail = 0 - m = dump.DumpMaster(None, o, outfile=cs) + m = dump.DumpMaster(None, o) self.cycle(m, b"nonexistent") o = dump.Options(client_replay=[p], kill=False) o.verbosity = 0 o.flow_detail = 0 - m = dump.DumpMaster(None, o, outfile=cs) + m = dump.DumpMaster(None, o) def test_read(self): with tutils.tmpdir() as t: @@ -172,20 +183,24 @@ class TestDumpMaster(mastertest.MasterTest): assert len(m.apps.apps) == 1 def test_replacements(self): - cs = StringIO() - o = dump.Options(replacements=[(".*", "content", "foo")]) + o = dump.Options( + replacements=[(".*", "content", "foo")], + tfile = StringIO(), + ) o.verbosity = 0 o.flow_detail = 0 - m = dump.DumpMaster(None, o, outfile=cs) + m = dump.DumpMaster(None, o) f = self.cycle(m, b"content") assert f.request.content == b"foo" def test_setheader(self): - cs = StringIO() - o = dump.Options(setheaders=[(".*", "one", "two")]) + o = dump.Options( + setheaders=[(".*", "one", "two")], + tfile=StringIO() + ) o.verbosity = 0 o.flow_detail = 0 - m = dump.DumpMaster(None, o, outfile=cs) + m = dump.DumpMaster(None, o) f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" @@ -212,7 +227,7 @@ class TestDumpMaster(mastertest.MasterTest): def test_write_err(self): tutils.raises( - dump.DumpError, + exceptions.OptionsError, self.mkmaster, None, outfile = ("nonexistentdir/foo", "wb") ) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 91342e58..1a07f74d 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -1,5 +1,3 @@ -import os.path - import mock import io @@ -887,35 +885,6 @@ class TestFlowMaster: fm.process_new_request(f) assert "killed" in f.error.msg - def test_stream(self): - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") - - def read(): - with open(p, "rb") as f: - r = flow.FlowReader(f) - return list(r.stream()) - - s = flow.State() - fm = flow.FlowMaster(None, None, s) - f = tutils.tflow(resp=True) - - with open(p, "ab") as tmpfile: - fm.start_stream(tmpfile, None) - fm.request(f) - fm.response(f) - fm.stop_stream() - - assert read()[0].response - - with open(p, "ab") as tmpfile: - f = tutils.tflow() - fm.start_stream(tmpfile, None) - fm.request(f) - fm.shutdown() - - assert not read()[1].response - class TestRequest: -- cgit v1.2.3 From ec6fbe9eb684bbf70db796876751078865a0e50f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 13 Jul 2016 21:40:13 -0700 Subject: make options explicit --- mitmproxy/console/master.py | 58 ++++++++++------------- mitmproxy/dump.py | 54 ++++++++-------------- mitmproxy/flow/__init__.py | 2 + mitmproxy/flow/options.py | 69 ++++++++++++++++++++++++++++ mitmproxy/options.py | 27 +++++++++-- mitmproxy/web/master.py | 62 ++++++++++--------------- test/mitmproxy/builtins/test_anticache.py | 2 +- test/mitmproxy/builtins/test_anticomp.py | 2 +- test/mitmproxy/builtins/test_stickyauth.py | 2 +- test/mitmproxy/builtins/test_stickycookie.py | 2 +- test/mitmproxy/console/test_master.py | 6 +-- test/mitmproxy/test_options.py | 30 ++++++++---- test/mitmproxy/test_web_master.py | 9 ++-- 13 files changed, 191 insertions(+), 134 deletions(-) create mode 100644 mitmproxy/flow/options.py diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 5c015033..bc373a2b 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -14,6 +14,7 @@ import traceback import weakref import urwid +from typing import Optional # noqa from mitmproxy import builtins from mitmproxy import contentviews @@ -21,7 +22,6 @@ from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import script -import mitmproxy.options from mitmproxy.console import flowlist from mitmproxy.console import flowview from mitmproxy.console import grideditor @@ -177,40 +177,26 @@ class ConsoleState(flow.State): self.add_flow_setting(flow, "marked", marked) -class Options(mitmproxy.options.Options): - attributes = [ - "app", - "app_domain", - "app_ip", - "anticache", - "anticomp", - "client_replay", - "eventlog", - "follow", - "keepserving", - "kill", - "intercept", - "limit", - "no_server", - "refresh_server_playback", - "rfile", - "scripts", - "showhost", - "replacements", - "rheaders", - "setheaders", - "server_replay", - "stickycookie", - "stickyauth", - "stream_large_bodies", - "verbosity", - "wfile", - "nopop", - "palette", - "palette_transparent", - "no_mouse", - "outfile", - ] +class Options(flow.options.Options): + def __init__( + self, + eventlog=False, # type: bool + follow=False, # type: bool + intercept=False, # type: bool + limit=None, # type: Optional[str] + palette=None, # type: Optional[str] + palette_transparent=False, # type: bool + no_mouse=False, # type: bool + **kwargs + ): + self.eventlog = eventlog + self.follow = follow + self.intercept = intercept + self.limit = limit + self.palette = palette + self.palette_transparent = palette_transparent + self.no_mouse = no_mouse + super(Options, self).__init__(**kwargs) class ConsoleMaster(flow.FlowMaster): @@ -221,6 +207,8 @@ class ConsoleMaster(flow.FlowMaster): self.addons.add(*builtins.default_addons()) self.stream_path = None + # This line is just for type hinting + self.options = self.options # type: Options self.options.errored.connect(self.options_error) if options.replacements: diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 296419db..90df6e1b 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -6,12 +6,14 @@ import traceback import click +from typing import Optional # noqa +import typing + from mitmproxy import contentviews from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import flow -from mitmproxy import options from mitmproxy import builtins from netlib import human from netlib import tcp @@ -22,40 +24,20 @@ class DumpError(Exception): pass -class Options(options.Options): - attributes = [ - "app", - "app_host", - "app_port", - "anticache", - "anticomp", - "client_replay", - "filtstr", - "flow_detail", - "keepserving", - "kill", - "no_server", - "nopop", - "refresh_server_playback", - "replacements", - "rfile", - "rheaders", - "setheaders", - "server_replay", - "scripts", - "showhost", - "stickycookie", - "stickyauth", - "stream_large_bodies", - "verbosity", - "outfile", - "replay_ignore_content", - "replay_ignore_params", - "replay_ignore_payload_params", - "replay_ignore_host", - - "tfile" - ] +class Options(flow.options.Options): + def __init__( + self, + filtstr=None, # type: Optional[str] + flow_detail=1, # type: int + keepserving=False, # type: bool + tfile=None, # type: Optional[typing.io.TextIO] + **kwargs + ): + self.filtstr = filtstr + self.flow_detail = flow_detail + self.keepserving = keepserving + self.tfile = tfile + super(Options, self).__init__(**kwargs) class DumpMaster(flow.FlowMaster): @@ -63,6 +45,8 @@ class DumpMaster(flow.FlowMaster): def __init__(self, server, options): flow.FlowMaster.__init__(self, options, server, flow.State()) self.addons.add(*builtins.default_addons()) + # This line is just for type hinting + self.options = self.options # type: Options self.o = options self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index 4c3bb828..caa17528 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -8,6 +8,7 @@ from mitmproxy.flow.modules import ( ServerPlaybackState ) from mitmproxy.flow.state import State, FlowView +from mitmproxy.flow import options # TODO: We may want to remove the imports from .modules and just expose "modules" @@ -17,4 +18,5 @@ __all__ = [ "FlowMaster", "AppRegistry", "ReplaceHooks", "SetHeaders", "StreamLargeBodies", "ClientPlaybackState", "ServerPlaybackState", "State", "FlowView", + "options", ] diff --git a/mitmproxy/flow/options.py b/mitmproxy/flow/options.py new file mode 100644 index 00000000..4bafad0f --- /dev/null +++ b/mitmproxy/flow/options.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import, print_function, division +from mitmproxy import options +from typing import Tuple, Optional, Sequence # noqa + +APP_HOST = "mitm.it" +APP_PORT = 80 + + +class Options(options.Options): + def __init__( + self, + # TODO: rename to onboarding_app_* + app=True, # type: bool + app_host=APP_HOST, # type: str + app_port=APP_PORT, # type: int + anticache=False, # type: bool + anticomp=False, # type: bool + client_replay=None, # type: Optional[str] + kill=False, # type: bool + no_server=False, # type: bool + nopop=False, # type: bool + refresh_server_playback=False, # type: bool + rfile=None, # type: Optional[str] + scripts=(), # type: Sequence[str] + showhost=False, # type: bool + replacements=(), # type: Sequence[Tuple[str, str, str]] + rheaders=(), # type: Sequence[str] + setheaders=(), # type: Sequence[Tuple[str, str, str]] + server_replay=None, # type: Optional[str] + stickycookie=None, # type: Optional[str] + stickyauth=None, # type: Optional[str] + stream_large_bodies=None, # type: Optional[str] + verbosity=1, # type: int + outfile=None, # type: Optional[str] + replay_ignore_content=False, # type: bool + replay_ignore_params=(), # type: Sequence[str] + replay_ignore_payload_params=(), # type: Sequence[str] + replay_ignore_host=False, # type: bool + ): + # We could replace all assignments with clever metaprogramming, + # but type hints are a much more valueable asset. + + self.app = app + self.app_host = app_host + self.app_port = app_port + self.anticache = anticache + self.anticomp = anticomp + self.client_replay = client_replay + self.kill = kill + self.no_server = no_server + self.nopop = nopop + self.refresh_server_playback = refresh_server_playback + self.rfile = rfile + self.scripts = scripts + self.showhost = showhost + self.replacements = replacements + self.rheaders = rheaders + self.setheaders = setheaders + self.server_replay = server_replay + self.stickycookie = stickycookie + self.stickyauth = stickyauth + self.stream_large_bodies = stream_large_bodies + self.verbosity = verbosity + self.outfile = outfile + self.replay_ignore_content = replay_ignore_content + self.replay_ignore_params = replay_ignore_params + self.replay_ignore_payload_params = replay_ignore_payload_params + self.replay_ignore_host = replay_ignore_host + super(Options, self).__init__() diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 5599185d..a124eaf6 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -14,14 +14,21 @@ class Options(object): exception, all changes are rolled back, the exception is suppressed, and the .errored signal is notified. """ + _initialized = False attributes = [] - def __init__(self, **kwargs): + def __new__(cls, *args, **kwargs): + # Initialize instance._opts before __init__ is called. + # This allows us to call super().__init__() last, which then sets + # ._initialized = True as the final operation. + instance = super(Options, cls).__new__(cls) + instance.__dict__["_opts"] = {} + return instance + + def __init__(self): self.__dict__["changed"] = blinker.Signal() self.__dict__["errored"] = blinker.Signal() - self.__dict__["_opts"] = dict([(i, None) for i in self.attributes]) - for k, v in kwargs.items(): - self._opts[k] = v + self.__dict__["_initialized"] = True @contextlib.contextmanager def rollback(self): @@ -48,6 +55,9 @@ class Options(object): raise AttributeError() def __setattr__(self, attr, value): + if not self._initialized: + self._opts[attr] = value + return if attr not in self._opts: raise KeyError("No such option: %s" % attr) with self.rollback(): @@ -71,4 +81,11 @@ class Options(object): return lambda x: self.__setattr__(attr, x) def __repr__(self): - return pprint.pformat(self._opts) + options = pprint.pformat(self._opts, indent=4).strip(" {}") + if "\n" in options: + options = "\n " + options + "\n" + return "{mod}.{cls}({{{options}}})".format( + mod=type(self).__module__, + cls=type(self).__name__, + options=options + ) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 7c775c32..75cc7746 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -6,11 +6,12 @@ import collections import tornado.httpserver import tornado.ioloop +from typing import Optional # noqa + from mitmproxy import builtins from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow -from mitmproxy import options from mitmproxy.web import app from netlib.http import authentication @@ -90,43 +91,26 @@ class WebState(flow.State): ) -class Options(options.Options): - attributes = [ - "app", - "app_domain", - "app_ip", - "anticache", - "anticomp", - "client_replay", - "eventlog", - "keepserving", - "kill", - "intercept", - "no_server", - "outfile", - "refresh_server_playback", - "rfile", - "scripts", - "showhost", - "replacements", - "rheaders", - "setheaders", - "server_replay", - "stickycookie", - "stickyauth", - "stream_large_bodies", - "verbosity", - "wfile", - "nopop", - - "wdebug", - "wport", - "wiface", - "wauthenticator", - "wsingleuser", - "whtpasswd", - ] - +class Options(flow.options.Options): + def __init__( + self, + wdebug=bool, # type: bool + wport=8081, # type: int + wiface="127.0.0.1", # type: str + wauthenticator=None, # type: Optional[authentication.PassMan] + wsingleuser=None, # type: Optional[str] + whtpasswd=None, # type: Optional[str] + **kwargs + ): + self.wdebug = wdebug + self.wport = wport + self.wiface = wiface + self.wauthenticator = wauthenticator + self.wsingleuser = wsingleuser + self.whtpasswd = whtpasswd + super(Options, self).__init__(**kwargs) + + # TODO: This doesn't belong here. def process_web_options(self, parser): if self.wsingleuser or self.whtpasswd: if self.wsingleuser: @@ -153,6 +137,8 @@ class WebMaster(flow.FlowMaster): self.app = app.Application( self, self.options.wdebug, self.options.wauthenticator ) + # This line is just for type hinting + self.options = self.options # type: Options if options.rfile: try: self.load_flows_file(options.rfile) diff --git a/test/mitmproxy/builtins/test_anticache.py b/test/mitmproxy/builtins/test_anticache.py index 5a00af03..127e1c1a 100644 --- a/test/mitmproxy/builtins/test_anticache.py +++ b/test/mitmproxy/builtins/test_anticache.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import anticache from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options class TestAntiCache(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_anticomp.py b/test/mitmproxy/builtins/test_anticomp.py index 6bfd54bb..601e56c8 100644 --- a/test/mitmproxy/builtins/test_anticomp.py +++ b/test/mitmproxy/builtins/test_anticomp.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import anticomp from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options class TestAntiComp(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_stickyauth.py b/test/mitmproxy/builtins/test_stickyauth.py index 9233f435..1e617402 100644 --- a/test/mitmproxy/builtins/test_stickyauth.py +++ b/test/mitmproxy/builtins/test_stickyauth.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import stickyauth from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options class TestStickyAuth(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py index e64ecb5b..9cf768df 100644 --- a/test/mitmproxy/builtins/test_stickycookie.py +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import stickycookie from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options from netlib import tutils as ntutils diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index d42863b3..b84e4c1c 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -111,14 +111,14 @@ def test_options(): class TestMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): + def mkmaster(self, **options): if "verbosity" not in options: options["verbosity"] = 0 - o = console.master.Options(filtstr=filt, **options) + o = console.master.Options(**options) return console.master.ConsoleMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py index 97db9430..cdb0d765 100644 --- a/test/mitmproxy/test_options.py +++ b/test/mitmproxy/test_options.py @@ -7,10 +7,10 @@ from netlib import tutils class TO(options.Options): - attributes = [ - "one", - "two" - ] + def __init__(self, one=None, two=None): + self.one = one + self.two = two + super(TO, self).__init__() def test_options(): @@ -19,8 +19,13 @@ def test_options(): assert o.two == "three" o.one = "one" assert o.one == "one" - tutils.raises("no such option", setattr, o, "nonexistent", "value") - tutils.raises("no such option", o.update, nonexistent = "value") + + with tutils.raises(TypeError): + TO(nonexistent = "value") + with tutils.raises("no such option"): + o.nonexistent = "value" + with tutils.raises("no such option"): + o.update(nonexistent = "value") rec = [] @@ -43,7 +48,8 @@ def test_setter(): f = o.setter("two") f("xxx") assert o.two == "xxx" - tutils.raises("no such option", o.setter, "nonexistent") + with tutils.raises("no such option"): + o.setter("nonexistent") def test_rollback(): @@ -61,7 +67,7 @@ def test_rollback(): def err(opts): if opts.one == "ten": - raise exceptions.OptionsError + raise exceptions.OptionsError() o.changed.connect(sub) o.changed.connect(err) @@ -73,3 +79,11 @@ def test_rollback(): assert len(rec) == 2 assert rec[0].one == "ten" assert rec[1].one == "two" + + +def test_repr(): + assert repr(TO()) == "test.mitmproxy.test_options.TO({'one': None, 'two': None})" + assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_options.TO({ + 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + 'two': None +})""" diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index f0fafe24..2ab440ce 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -3,15 +3,12 @@ from . import mastertest class TestWebMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): - o = master.Options( - filtstr=filt, - **options - ) + def mkmaster(self, **options): + o = master.Options(**options) return master.WebMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i -- cgit v1.2.3 From f84a23f11d869ca5132e0388345bd19937486628 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 13 Jul 2016 22:01:16 -0700 Subject: minor fixes --- mitmproxy/dump.py | 2 +- mitmproxy/flow/options.py | 2 +- test/mitmproxy/builtins/test_stream.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 90df6e1b..274e01f3 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -7,7 +7,7 @@ import traceback import click from typing import Optional # noqa -import typing +import typing # noqa from mitmproxy import contentviews from mitmproxy import controller diff --git a/mitmproxy/flow/options.py b/mitmproxy/flow/options.py index 4bafad0f..eccba5b1 100644 --- a/mitmproxy/flow/options.py +++ b/mitmproxy/flow/options.py @@ -31,7 +31,7 @@ class Options(options.Options): stickyauth=None, # type: Optional[str] stream_large_bodies=None, # type: Optional[str] verbosity=1, # type: int - outfile=None, # type: Optional[str] + outfile=None, # type: Tuple[str, str] replay_ignore_content=False, # type: bool replay_ignore_params=(), # type: Sequence[str] replay_ignore_payload_params=(), # type: Sequence[str] diff --git a/test/mitmproxy/builtins/test_stream.py b/test/mitmproxy/builtins/test_stream.py index 54e4f7d9..edaa41d2 100644 --- a/test/mitmproxy/builtins/test_stream.py +++ b/test/mitmproxy/builtins/test_stream.py @@ -7,7 +7,7 @@ import os.path from mitmproxy.builtins import stream from mitmproxy.flow import master, FlowReader from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options class TestStream(mastertest.MasterTest): -- cgit v1.2.3 From 126625584251d6a246ba46943cfa71d5a57fbdda Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 14 Jul 2016 13:07:27 +0530 Subject: Re-add deletion of stickycookies (#1355) This was originally added in #1324 (fixing issue #1096) but got lost during "the big refactor" (#1352) --- mitmproxy/builtins/stickycookie.py | 18 ++++++++++++++---- test/mitmproxy/builtins/test_stickycookie.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mitmproxy/builtins/stickycookie.py b/mitmproxy/builtins/stickycookie.py index 5913976c..dc699bb4 100644 --- a/mitmproxy/builtins/stickycookie.py +++ b/mitmproxy/builtins/stickycookie.py @@ -46,10 +46,20 @@ class StickyCookie: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with # valid RFC 822/1123 datetime specifications for expiry. Sigh. - a = ckey(attrs, flow) - if domain_match(flow.request.host, a[0]): - b = attrs.with_insert(0, name, value) - self.jar[a][name] = b + dom_port_path = ckey(attrs, flow) + + if domain_match(flow.request.host, dom_port_path[0]): + if cookies.is_expired(attrs): + # Remove the cookie from jar + self.jar[dom_port_path].pop(name, None) + + # If all cookies of a dom_port_path have been removed + # then remove it from the jar itself + if not self.jar[dom_port_path]: + self.jar.pop(dom_port_path, None) + else: + b = attrs.with_insert(0, name, value) + self.jar[dom_port_path][name] = b def request(self, flow): if self.flt: diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py index 9cf768df..b8d703bd 100644 --- a/test/mitmproxy/builtins/test_stickycookie.py +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -112,6 +112,16 @@ class TestStickyCookie(mastertest.MasterTest): assert len(sc.jar[googlekey].keys()) == 1 assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" + def test_response_delete(self): + s, m, sc = self.mk() + + # Test that a cookie is be deleted + # by setting the expire time in the past + f = self._response(s, m, sc, "duffer=zafar; Path=/", "www.google.com") + f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT" + self.invoke(m, "response", f) + assert not sc.jar.keys() + def test_request(self): s, m, sc = self.mk() -- cgit v1.2.3 From a3a22fba337fc4ac750b8c18663233920a0d646b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 13:51:00 +1200 Subject: First-order integration of scripts addon --- mitmproxy/builtins/__init__.py | 2 + mitmproxy/builtins/script.py | 156 +++++++++++++++++++++ mitmproxy/controller.py | 12 +- mitmproxy/dump.py | 8 -- mitmproxy/exceptions.py | 4 + mitmproxy/flow/master.py | 98 ++----------- test/mitmproxy/builtins/test_script.py | 136 ++++++++++++++++++ test/mitmproxy/data/addonscripts/duplicate_flow.py | 6 + test/mitmproxy/data/addonscripts/error.py | 7 + test/mitmproxy/data/addonscripts/recorder.py | 18 +++ test/mitmproxy/data/addonscripts/stream_modify.py | 8 ++ .../data/addonscripts/tcp_stream_modify.py | 5 + test/mitmproxy/test_dump.py | 4 +- test/mitmproxy/test_flow.py | 64 +-------- test/mitmproxy/test_script.py | 13 -- test/mitmproxy/test_server.py | 22 +-- 16 files changed, 379 insertions(+), 184 deletions(-) create mode 100644 mitmproxy/builtins/script.py create mode 100644 test/mitmproxy/builtins/test_script.py create mode 100644 test/mitmproxy/data/addonscripts/duplicate_flow.py create mode 100644 test/mitmproxy/data/addonscripts/error.py create mode 100644 test/mitmproxy/data/addonscripts/recorder.py create mode 100644 test/mitmproxy/data/addonscripts/stream_modify.py create mode 100644 test/mitmproxy/data/addonscripts/tcp_stream_modify.py delete mode 100644 test/mitmproxy/test_script.py diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index 8021c20f..6b357902 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -4,6 +4,7 @@ from mitmproxy.builtins import anticache from mitmproxy.builtins import anticomp from mitmproxy.builtins import stickyauth from mitmproxy.builtins import stickycookie +from mitmproxy.builtins import script from mitmproxy.builtins import stream @@ -13,5 +14,6 @@ def default_addons(): anticomp.AntiComp(), stickyauth.StickyAuth(), stickycookie.StickyCookie(), + script.ScriptLoader(), stream.Stream(), ] diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py new file mode 100644 index 00000000..015adef9 --- /dev/null +++ b/mitmproxy/builtins/script.py @@ -0,0 +1,156 @@ +from __future__ import absolute_import, print_function, division + +import contextlib +import os +import shlex +import sys +import traceback +import copy + +from mitmproxy import exceptions +from mitmproxy import controller +from mitmproxy import ctx + + +import watchdog.events +# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths. +# We use the PollingObserver instead. +if sys.platform == 'darwin': # pragma: no cover + from watchdog.observers.polling import PollingObserver as Observer +else: + from watchdog.observers import Observer + + +def parse_command(command): + """ + Returns a (path, args) tuple. + """ + if not command or not command.strip(): + raise exceptions.AddonError("Empty script command.") + # Windows: escape all backslashes in the path. + if os.name == "nt": # pragma: no cover + backslashes = shlex.split(command, posix=False)[0].count("\\") + command = command.replace("\\", "\\\\", backslashes) + args = shlex.split(command) # pragma: no cover + args[0] = os.path.expanduser(args[0]) + if not os.path.exists(args[0]): + raise exceptions.AddonError( + ("Script file not found: %s.\r\n" + "If your script path contains spaces, " + "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % + args[0]) + elif os.path.isdir(args[0]): + raise exceptions.AddonError("Not a file: %s" % args[0]) + return args[0], args[1:] + + +@contextlib.contextmanager +def scriptenv(path, args): + oldargs = sys.argv + sys.argv = [path] + args + script_dir = os.path.dirname(os.path.abspath(path)) + sys.path.append(script_dir) + try: + yield + except Exception: + _, _, tb = sys.exc_info() + scriptdir = os.path.dirname(os.path.abspath(path)) + for i, s in enumerate(reversed(traceback.extract_tb(tb))): + if not os.path.abspath(s[0]).startswith(scriptdir): + break + else: + tb = tb.tb_next + ctx.log.warn("".join(traceback.format_tb(tb))) + finally: + sys.argv = oldargs + sys.path.pop() + + +def load_script(path, args): + ns = {'__file__': os.path.abspath(path)} + with scriptenv(path, args): + with open(path, "rb") as f: + code = compile(f.read(), path, 'exec') + exec(code, ns, ns) + return ns + + +class ReloadHandler(watchdog.events.FileSystemEventHandler): + def __init__(self, callback, master, options): + self.callback = callback + self.master, self.options = master, options + + def on_modified(self, event): + self.callback(self.master, self.options) + + def on_created(self, event): + self.callback(self.master, self.options) + + +class Script: + """ + An addon that manages a single script. + """ + def __init__(self, command): + self.name = command + + self.command = command + self.path, self.args = parse_command(command) + self.ns = None + self.observer = None + + for i in controller.Events: + def mkprox(): + evt = i + + def prox(*args, **kwargs): + self.run(evt, *args, **kwargs) + return prox + setattr(self, i, mkprox()) + + def run(self, name, *args, **kwargs): + # It's possible for ns to be un-initialised if we failed during + # configure + if self.ns is not None: + func = self.ns.get(name) + if func: + with scriptenv(self.path, self.args): + func(*args, **kwargs) + + def reload(self, master, options): + with master.handlecontext(): + self.ns = None + self.configure(options) + + def configure(self, options): + if not self.observer: + self.observer = Observer() + # Bind the handler to the real underlying master object + self.observer.schedule( + ReloadHandler( + self.reload, + ctx.master, + copy.copy(options), + ), + os.path.dirname(self.path) or "." + ) + self.observer.start() + if not self.ns: + self.ns = load_script(self.path, self.args) + self.run("configure", options) + + +class ScriptLoader(): + """ + An addon that manages loading scripts from options. + """ + def configure(self, options): + for s in options.scripts or []: + if not ctx.master.addons.has_addon(s): + ctx.log.info("Loading script: %s" % s) + sc = Script(s) + ctx.master.addons.add(sc) + for a in ctx.master.addons.chain: + if isinstance(a, Script): + if a.name not in options.scripts or []: + ctx.master.addons.remove(a) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 2f0c8bf2..464842b6 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -44,7 +44,17 @@ class Log(object): def __call__(self, text, level="info"): self.master.add_event(text, level) - # We may want to add .log(), .warn() etc. here at a later point in time + def debug(self, txt): + self(txt, "debug") + + def info(self, txt): + self(txt, "info") + + def warn(self, txt): + self(txt, "warn") + + def error(self, txt): + self(txt, "error") class Master(object): diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 274e01f3..999a709a 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -93,13 +93,6 @@ class DumpMaster(flow.FlowMaster): not options.keepserving ) - scripts = options.scripts or [] - for command in scripts: - try: - self.load_script(command, use_reloader=True) - except exceptions.ScriptException as e: - raise DumpError(str(e)) - if options.rfile: try: self.load_flows_file(options.rfile) @@ -335,6 +328,5 @@ class DumpMaster(flow.FlowMaster): def run(self): # pragma: no cover if self.options.rfile and not self.options.keepserving: - self.unload_scripts() # make sure to trigger script unload events. return super(DumpMaster, self).run() diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 282784b6..3b41fe1c 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -99,3 +99,7 @@ class ControlException(ProxyException): class OptionsError(Exception): pass + + +class AddonError(Exception): + pass diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 27ceee87..dbb19ed9 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -9,7 +9,6 @@ import netlib.exceptions from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import models -from mitmproxy import script from mitmproxy.flow import io from mitmproxy.flow import modules from mitmproxy.onboarding import app @@ -35,8 +34,6 @@ class FlowMaster(controller.Master): self.server_playback = None # type: Optional[modules.ServerPlaybackState] self.client_playback = None # type: Optional[modules.ClientPlaybackState] self.kill_nonreplay = False - self.scripts = [] # type: List[script.Script] - self.pause_scripts = False self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] self.refresh_server_playback = False @@ -60,44 +57,6 @@ class FlowMaster(controller.Master): level: debug, info, error """ - def unload_scripts(self): - for s in self.scripts[:]: - self.unload_script(s) - - def unload_script(self, script_obj): - try: - script_obj.unload() - except script.ScriptException as e: - self.add_event("Script error:\n" + str(e), "error") - script.reloader.unwatch(script_obj) - self.scripts.remove(script_obj) - - def load_script(self, command, use_reloader=False): - """ - Loads a script. - - Raises: - ScriptException - """ - s = script.Script(command) - s.load() - if use_reloader: - s.reply = controller.DummyReply() - script.reloader.watch(s, lambda: self.event_queue.put(("script_change", s))) - self.scripts.append(s) - - def _run_single_script_hook(self, script_obj, name, *args, **kwargs): - if script_obj and not self.pause_scripts: - try: - script_obj.run(name, *args, **kwargs) - except script.ScriptException as e: - self.add_event("Script error:\n{}".format(e), "error") - - def run_scripts(self, name, msg): - for script_obj in self.scripts: - if not msg.reply.acked: - self._run_single_script_hook(script_obj, name, msg) - def get_ignore_filter(self): return self.server.config.check_ignore.patterns @@ -298,11 +257,11 @@ class FlowMaster(controller.Master): if not pb and self.kill_nonreplay: f.kill(self) - def replay_request(self, f, block=False, run_scripthooks=True): + def replay_request(self, f, block=False): """ Returns None if successful, or error message if not. """ - if f.live and run_scripthooks: + if f.live: return "Can't replay live request." if f.intercepted: return "Can't replay while intercepting..." @@ -319,7 +278,7 @@ class FlowMaster(controller.Master): rt = http_replay.RequestReplayThread( self.server.config, f, - self.event_queue if run_scripthooks else False, + self.event_queue, self.should_exit ) rt.start() # pragma: no cover @@ -332,28 +291,27 @@ class FlowMaster(controller.Master): @controller.handler def clientconnect(self, root_layer): - self.run_scripts("clientconnect", root_layer) + pass @controller.handler def clientdisconnect(self, root_layer): - self.run_scripts("clientdisconnect", root_layer) + pass @controller.handler def serverconnect(self, server_conn): - self.run_scripts("serverconnect", server_conn) + pass @controller.handler def serverdisconnect(self, server_conn): - self.run_scripts("serverdisconnect", server_conn) + pass @controller.handler def next_layer(self, top_layer): - self.run_scripts("next_layer", top_layer) + pass @controller.handler def error(self, f): self.state.update_flow(f) - self.run_scripts("error", f) if self.client_playback: self.client_playback.clear(f) return f @@ -381,8 +339,6 @@ class FlowMaster(controller.Master): self.setheaders.run(f) if not f.reply.acked: self.process_new_request(f) - if not f.reply.acked: - self.run_scripts("request", f) return f @controller.handler @@ -393,7 +349,6 @@ class FlowMaster(controller.Master): except netlib.exceptions.HttpException: f.reply.kill() return - self.run_scripts("responseheaders", f) return f @controller.handler @@ -404,7 +359,6 @@ class FlowMaster(controller.Master): self.replacehooks.run(f) if not f.reply.acked: self.setheaders.run(f) - self.run_scripts("response", f) if not f.reply.acked: if self.client_playback: self.client_playback.clear(f) @@ -416,46 +370,15 @@ class FlowMaster(controller.Master): def handle_accept_intercept(self, f): self.state.update_flow(f) - @controller.handler - def script_change(self, s): - """ - Handle a script whose contents have been changed on the file system. - - Args: - s (script.Script): the changed script - - Returns: - True, if reloading was successful. - False, otherwise. - """ - ok = True - # We deliberately do not want to fail here. - # In the worst case, we have an "empty" script object. - try: - s.unload() - except script.ScriptException as e: - ok = False - self.add_event('Error reloading "{}":\n{}'.format(s.path, e), 'error') - try: - s.load() - except script.ScriptException as e: - ok = False - self.add_event('Error reloading "{}":\n{}'.format(s.path, e), 'error') - else: - self.add_event('"{}" reloaded.'.format(s.path), 'info') - return ok - @controller.handler def tcp_open(self, flow): # TODO: This would break mitmproxy currently. # self.state.add_flow(flow) self.active_flows.add(flow) - self.run_scripts("tcp_open", flow) @controller.handler def tcp_message(self, flow): - # type: (TCPFlow) -> None - self.run_scripts("tcp_message", flow) + pass @controller.handler def tcp_error(self, flow): @@ -463,13 +386,10 @@ class FlowMaster(controller.Master): repr(flow.server_conn.address), flow.error ), "info") - self.run_scripts("tcp_error", flow) @controller.handler def tcp_close(self, flow): self.active_flows.discard(flow) - self.run_scripts("tcp_close", flow) def shutdown(self): super(FlowMaster, self).shutdown() - self.unload_scripts() diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py new file mode 100644 index 00000000..d3366189 --- /dev/null +++ b/test/mitmproxy/builtins/test_script.py @@ -0,0 +1,136 @@ +import time + +from mitmproxy.builtins import script +from mitmproxy import exceptions +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options + +from .. import tutils, mastertest + + +class TestParseCommand: + def test_empty_command(self): + with tutils.raises(exceptions.AddonError): + script.parse_command("") + + with tutils.raises(exceptions.AddonError): + script.parse_command(" ") + + def test_no_script_file(self): + with tutils.raises("not found"): + script.parse_command("notfound") + + with tutils.tmpdir() as dir: + with tutils.raises("not a file"): + script.parse_command(dir) + + def test_parse_args(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", []) + assert script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"]) + assert script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"]) + + @tutils.skip_not_windows + def test_parse_windows(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", []) + assert script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", 'foo \\ bar', []) + + +def test_load_script(): + ns = script.load_script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ), [] + ) + assert ns["configure"] + + +class RecordingMaster(master.FlowMaster): + def __init__(self, *args, **kwargs): + master.FlowMaster.__init__(self, *args, **kwargs) + self.event_log = [] + + def add_event(self, e, level): + self.event_log.append((level, e)) + + +class TestScript(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ) + ) + m.addons.add(sc) + assert sc.ns["call_log"] == [("configure", (options.Options(),), {})] + + sc.ns["call_log"] = [] + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + recf = sc.ns["call_log"][0] + assert recf[0] == "request" + + def test_reload(self): + s = state.State() + m = RecordingMaster(options.Options(), None, s) + with tutils.tmpdir(): + with open("foo.py", "w"): + pass + sc = script.Script("foo.py") + m.addons.add(sc) + + for _ in range(100): + with open("foo.py", "a") as f: + f.write(".") + time.sleep(0.1) + if m.event_log: + return + raise AssertionError("Change event not detected.") + + def test_exception(self): + s = state.State() + m = RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path("data/addonscripts/error.py") + ) + m.addons.add(sc) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert m.event_log[0][0] == "warn" + + def test_duplicate_flow(self): + s = state.State() + fm = master.FlowMaster(None, None, s) + fm.addons.add( + script.Script( + tutils.test_data.path("data/addonscripts/duplicate_flow.py") + ) + ) + f = tutils.tflow() + fm.request(f) + assert fm.state.flow_count() == 2 + assert not fm.state.view[0].request.is_replay + assert fm.state.view[1].request.is_replay + + +class TestScriptLoader(mastertest.MasterTest): + def test_simple(self): + s = state.State() + o = options.Options(scripts=[]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + assert len(m.addons) == 1 + o.update( + scripts = [ + tutils.test_data.path("data/addonscripts/recorder.py") + ] + ) + assert len(m.addons) == 2 + o.update(scripts = []) + assert len(m.addons) == 1 diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py new file mode 100644 index 00000000..b466423c --- /dev/null +++ b/test/mitmproxy/data/addonscripts/duplicate_flow.py @@ -0,0 +1,6 @@ +from mitmproxy import ctx + + +def request(flow): + f = ctx.master.duplicate_flow(flow) + ctx.master.replay_request(f, block=True) diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py new file mode 100644 index 00000000..8ece9fce --- /dev/null +++ b/test/mitmproxy/data/addonscripts/error.py @@ -0,0 +1,7 @@ + +def mkerr(): + raise ValueError("Error!") + + +def request(flow): + mkerr() diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py new file mode 100644 index 00000000..728203e3 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/recorder.py @@ -0,0 +1,18 @@ +from mitmproxy import controller +from mitmproxy import ctx + +call_log = [] + +# Keep a log of all possible event calls +evts = list(controller.Events) + ["configure"] +for i in evts: + def mkprox(): + evt = i + + def prox(*args, **kwargs): + lg = (evt, args, kwargs) + if evt != "log": + ctx.log.info(str(lg)) + call_log.append(lg) + return prox + globals()[i] = mkprox() diff --git a/test/mitmproxy/data/addonscripts/stream_modify.py b/test/mitmproxy/data/addonscripts/stream_modify.py new file mode 100644 index 00000000..bc616342 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/stream_modify.py @@ -0,0 +1,8 @@ + +def modify(chunks): + for chunk in chunks: + yield chunk.replace(b"foo", b"bar") + + +def responseheaders(flow): + flow.response.stream = modify diff --git a/test/mitmproxy/data/addonscripts/tcp_stream_modify.py b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py new file mode 100644 index 00000000..af4ccf7e --- /dev/null +++ b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py @@ -0,0 +1,5 @@ + +def tcp_message(flow): + message = flow.messages[-1] + if not message.from_client: + message.content = message.content.replace(b"foo", b"bar") diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 9686be84..201386e3 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -245,12 +245,12 @@ class TestDumpMaster(mastertest.MasterTest): assert "XRESPONSE" in ret assert "XCLIENTDISCONNECT" in ret tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["nonexistent"] ) tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["starterr.py"] ) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 1a07f74d..c58a9703 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -5,7 +5,7 @@ import netlib.utils from netlib.http import Headers from mitmproxy import filt, controller, flow from mitmproxy.contrib import tnetstring -from mitmproxy.exceptions import FlowReadException, ScriptException +from mitmproxy.exceptions import FlowReadException from mitmproxy.models import Error from mitmproxy.models import Flow from mitmproxy.models import HTTPFlow @@ -674,21 +674,6 @@ class TestSerialize: class TestFlowMaster: - def test_load_script(self): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.unload_scripts() - with tutils.raises(ScriptException): - fm.load_script("nonexistent") - try: - fm.load_script(tutils.test_data.path("data/scripts/starterr.py")) - except ScriptException as e: - assert "ValueError" in str(e) - assert len(fm.scripts) == 0 - def test_getset_ignore(self): p = mock.Mock() p.config.check_ignore = HostMatcher() @@ -708,51 +693,7 @@ class TestFlowMaster: assert "intercepting" in fm.replay_request(f) f.live = True - assert "live" in fm.replay_request(f, run_scripthooks=True) - - def test_script_reqerr(self): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - fm.load_script(tutils.test_data.path("data/scripts/reqerr.py")) - f = tutils.tflow() - fm.clientconnect(f.client_conn) - assert fm.request(f) - - def test_script(self): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - f = tutils.tflow(resp=True) - - f.client_conn.acked = False - fm.clientconnect(f.client_conn) - assert fm.scripts[0].ns["log"][-1] == "clientconnect" - f.server_conn.acked = False - fm.serverconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "serverconnect" - f.reply.acked = False - fm.request(f) - assert fm.scripts[0].ns["log"][-1] == "request" - f.reply.acked = False - fm.response(f) - assert fm.scripts[0].ns["log"][-1] == "response" - # load second script - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - assert len(fm.scripts) == 2 - f.server_conn.reply.acked = False - fm.clientdisconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" - assert fm.scripts[1].ns["log"][-1] == "clientdisconnect" - - # unload first script - fm.unload_scripts() - assert len(fm.scripts) == 0 - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - - f.error = tutils.terr() - f.reply.acked = False - fm.error(f) - assert fm.scripts[0].ns["log"][-1] == "error" + assert "live" in fm.replay_request(f) def test_duplicate_flow(self): s = flow.State() @@ -789,7 +730,6 @@ class TestFlowMaster: f.error.reply = controller.DummyReply() fm.error(f) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) fm.shutdown() def test_client_playback(self): diff --git a/test/mitmproxy/test_script.py b/test/mitmproxy/test_script.py deleted file mode 100644 index 1e8220f1..00000000 --- a/test/mitmproxy/test_script.py +++ /dev/null @@ -1,13 +0,0 @@ -from mitmproxy import flow -from . import tutils - - -def test_duplicate_flow(): - s = flow.State() - fm = flow.FlowMaster(None, None, s) - fm.load_script(tutils.test_data.path("data/scripts/duplicate_flow.py")) - f = tutils.tflow() - fm.request(f) - assert fm.state.flow_count() == 2 - assert not fm.state.view[0].request.is_replay - assert fm.state.view[1].request.is_replay diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 9dd8b79c..b1ca6910 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -13,6 +13,7 @@ from netlib.http import authentication, http1 from netlib.tutils import raises from pathod import pathoc, pathod +from mitmproxy.builtins import script from mitmproxy import controller from mitmproxy.proxy.config import HostMatcher from mitmproxy.models import Error, HTTPResponse, HTTPFlow @@ -287,10 +288,13 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.master.set_stream_large_bodies(None) def test_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/stream_modify.py")) + s = script.Script( + tutils.test_data.path("data/addonscripts/stream_modify.py") + ) + self.master.addons.add(s) d = self.pathod('200:b"foo"') - assert d.content == b"bar" - self.master.unload_scripts() + assert d.content == "bar" + self.master.addons.remove(s) class TestHTTPAuth(tservers.HTTPProxyTest): @@ -512,15 +516,15 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): ssl = False def test_tcp_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/tcp_stream_modify.py")) - + s = script.Script( + tutils.test_data.path("data/addonscripts/tcp_stream_modify.py") + ) + self.master.addons.add(s) self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() - - assert d.content == b"bar" - - self.master.unload_scripts() + assert d.content == "bar" + self.master.addons.remove(s) class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): -- cgit v1.2.3 From a6821aad8e9296640c3efd4275e8922dd7c6e43b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 14:39:07 +1200 Subject: Zap old scripts infrastructure, fix concurrency tests --- mitmproxy/flow/master.py | 3 - mitmproxy/script/__init__.py | 6 - mitmproxy/script/reloader.py | 47 ------- mitmproxy/script/script.py | 136 --------------------- test/mitmproxy/builtins/test_script.py | 13 +- .../data/addonscripts/concurrent_decorator.py | 6 + .../data/addonscripts/concurrent_decorator_err.py | 6 + .../mitmproxy/data/scripts/concurrent_decorator.py | 7 -- .../data/scripts/concurrent_decorator_err.py | 6 - test/mitmproxy/mastertest.py | 10 ++ test/mitmproxy/script/test_concurrent.py | 43 +++++-- 11 files changed, 55 insertions(+), 228 deletions(-) delete mode 100644 mitmproxy/script/reloader.py delete mode 100644 mitmproxy/script/script.py create mode 100644 test/mitmproxy/data/addonscripts/concurrent_decorator.py create mode 100644 test/mitmproxy/data/addonscripts/concurrent_decorator_err.py delete mode 100644 test/mitmproxy/data/scripts/concurrent_decorator.py delete mode 100644 test/mitmproxy/data/scripts/concurrent_decorator_err.py diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index dbb19ed9..aa09e109 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -390,6 +390,3 @@ class FlowMaster(controller.Master): @controller.handler def tcp_close(self, flow): self.active_flows.discard(flow) - - def shutdown(self): - super(FlowMaster, self).shutdown() diff --git a/mitmproxy/script/__init__.py b/mitmproxy/script/__init__.py index 9a3985ab..e75f282a 100644 --- a/mitmproxy/script/__init__.py +++ b/mitmproxy/script/__init__.py @@ -1,11 +1,5 @@ -from . import reloader from .concurrent import concurrent -from .script import Script -from ..exceptions import ScriptException __all__ = [ - "Script", "concurrent", - "ScriptException", - "reloader" ] diff --git a/mitmproxy/script/reloader.py b/mitmproxy/script/reloader.py deleted file mode 100644 index 857d76cd..00000000 --- a/mitmproxy/script/reloader.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import absolute_import, print_function, division - -import os - -from watchdog.events import RegexMatchingEventHandler - -from watchdog.observers.polling import PollingObserver as Observer -# We occasionally have watchdog errors on Windows, Linux and Mac when using the native observers. -# After reading through the watchdog source code and issue tracker, -# we may want to replace this with a very simple implementation of our own. - -_observers = {} - - -def watch(script, callback): - if script in _observers: - raise RuntimeError("Script already observed") - script_dir = os.path.dirname(os.path.abspath(script.path)) - script_name = os.path.basename(script.path) - event_handler = _ScriptModificationHandler(callback, filename=script_name) - observer = Observer() - observer.schedule(event_handler, script_dir) - observer.start() - _observers[script] = observer - - -def unwatch(script): - observer = _observers.pop(script, None) - if observer: - observer.stop() - observer.join() - - -class _ScriptModificationHandler(RegexMatchingEventHandler): - - def __init__(self, callback, filename='.*'): - - super(_ScriptModificationHandler, self).__init__( - ignore_directories=True, - regexes=['.*' + filename] - ) - self.callback = callback - - def on_modified(self, event): - self.callback() - -__all__ = ["watch", "unwatch"] diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py deleted file mode 100644 index db4909ca..00000000 --- a/mitmproxy/script/script.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -The script object representing mitmproxy inline scripts. -Script objects know nothing about mitmproxy or mitmproxy's API - this knowledge is provided -by the mitmproxy-specific ScriptContext. -""" -# Do not import __future__ here, this would apply transitively to the inline scripts. -from __future__ import absolute_import, print_function, division - -import os -import shlex -import sys -import contextlib - -import six -from typing import List # noqa - -from mitmproxy import exceptions - - -@contextlib.contextmanager -def scriptenv(path, args): - # type: (str, List[str]) -> None - oldargs = sys.argv - script_dir = os.path.dirname(os.path.abspath(path)) - - sys.argv = [path] + args - sys.path.append(script_dir) - try: - yield - finally: - sys.argv = oldargs - sys.path.pop() - - -class Script(object): - """ - Script object representing an inline script. - """ - - def __init__(self, command): - self.command = command - self.path, self.args = self.parse_command(command) - self.ns = None - - def __enter__(self): - self.load() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_val: - return False # re-raise the exception - self.unload() - - @staticmethod - def parse_command(command): - # type: (str) -> Tuple[str,List[str]] - """ - Returns a (path, args) tuple. - """ - if not command or not command.strip(): - raise exceptions.ScriptException("Empty script command.") - # Windows: escape all backslashes in the path. - if os.name == "nt": # pragma: no cover - backslashes = shlex.split(command, posix=False)[0].count("\\") - command = command.replace("\\", "\\\\", backslashes) - args = shlex.split(command) # pragma: no cover - args[0] = os.path.expanduser(args[0]) - if not os.path.exists(args[0]): - raise exceptions.ScriptException( - ("Script file not found: %s.\r\n" - "If your script path contains spaces, " - "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % - args[0]) - elif os.path.isdir(args[0]): - raise exceptions.ScriptException("Not a file: %s" % args[0]) - return args[0], args[1:] - - def load(self): - """ - Loads an inline script. - - Returns: - The return value of self.run("start", ...) - - Raises: - ScriptException on failure - """ - if self.ns is not None: - raise exceptions.ScriptException("Script is already loaded") - self.ns = {'__file__': os.path.abspath(self.path)} - - with scriptenv(self.path, self.args): - try: - with open(self.path) as f: - code = compile(f.read(), self.path, 'exec') - exec(code, self.ns, self.ns) - except Exception: - six.reraise( - exceptions.ScriptException, - exceptions.ScriptException.from_exception_context(), - sys.exc_info()[2] - ) - return self.run("start") - - def unload(self): - try: - return self.run("done") - finally: - self.ns = None - - def run(self, name, *args, **kwargs): - """ - Runs an inline script hook. - - Returns: - The return value of the method. - None, if the script does not provide the method. - - Raises: - ScriptException if there was an exception. - """ - if self.ns is None: - raise exceptions.ScriptException("Script not loaded.") - f = self.ns.get(name) - if f: - try: - with scriptenv(self.path, self.args): - return f(*args, **kwargs) - except Exception: - six.reraise( - exceptions.ScriptException, - exceptions.ScriptException.from_exception_context(), - sys.exc_info()[2] - ) - else: - return None diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index d3366189..2447c8ea 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -47,15 +47,6 @@ def test_load_script(): assert ns["configure"] -class RecordingMaster(master.FlowMaster): - def __init__(self, *args, **kwargs): - master.FlowMaster.__init__(self, *args, **kwargs) - self.event_log = [] - - def add_event(self, e, level): - self.event_log.append((level, e)) - - class TestScript(mastertest.MasterTest): def test_simple(self): s = state.State() @@ -77,7 +68,7 @@ class TestScript(mastertest.MasterTest): def test_reload(self): s = state.State() - m = RecordingMaster(options.Options(), None, s) + m = mastertest.RecordingMaster(options.Options(), None, s) with tutils.tmpdir(): with open("foo.py", "w"): pass @@ -94,7 +85,7 @@ class TestScript(mastertest.MasterTest): def test_exception(self): s = state.State() - m = RecordingMaster(options.Options(), None, s) + m = mastertest.RecordingMaster(options.Options(), None, s) sc = script.Script( tutils.test_data.path("data/addonscripts/error.py") ) diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator.py b/test/mitmproxy/data/addonscripts/concurrent_decorator.py new file mode 100644 index 00000000..a56c2af1 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator.py @@ -0,0 +1,6 @@ +import time +from mitmproxy.script import concurrent + +@concurrent +def request(flow): + time.sleep(0.1) diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py new file mode 100644 index 00000000..756869c8 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py @@ -0,0 +1,6 @@ +from mitmproxy.script import concurrent + + +@concurrent +def start(): + pass diff --git a/test/mitmproxy/data/scripts/concurrent_decorator.py b/test/mitmproxy/data/scripts/concurrent_decorator.py deleted file mode 100644 index 162c00f4..00000000 --- a/test/mitmproxy/data/scripts/concurrent_decorator.py +++ /dev/null @@ -1,7 +0,0 @@ -import time -from mitmproxy.script import concurrent - - -@concurrent -def request(flow): - time.sleep(0.1) diff --git a/test/mitmproxy/data/scripts/concurrent_decorator_err.py b/test/mitmproxy/data/scripts/concurrent_decorator_err.py deleted file mode 100644 index 756869c8..00000000 --- a/test/mitmproxy/data/scripts/concurrent_decorator_err.py +++ /dev/null @@ -1,6 +0,0 @@ -from mitmproxy.script import concurrent - - -@concurrent -def start(): - pass diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 9754d3a9..240f6a73 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -3,6 +3,7 @@ import mock from . import tutils import netlib.tutils +from mitmproxy.flow import master from mitmproxy import flow, proxy, models, controller @@ -39,3 +40,12 @@ class MasterTest: t = tutils.tflow(resp=True) fw.add(t) f.close() + + +class RecordingMaster(master.FlowMaster): + def __init__(self, *args, **kwargs): + master.FlowMaster.__init__(self, *args, **kwargs) + self.event_log = [] + + def add_event(self, e, level): + self.event_log.append((level, e)) diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 57eeca19..d5243bcb 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,28 +1,47 @@ -from mitmproxy.script import Script from test.mitmproxy import tutils from mitmproxy import controller +from mitmproxy.builtins import script +from mitmproxy import options +from mitmproxy.flow import master +from mitmproxy.flow import state import time +from .. import mastertest, tutils class Thing: def __init__(self): self.reply = controller.DummyReply() + self.live = True -@tutils.skip_appveyor -def test_concurrent(): - with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py")) as s: - f1, f2 = Thing(), Thing() - s.run("request", f1) - s.run("request", f2) +class TestConcurrent(mastertest.MasterTest): + @tutils.skip_appveyor + def test_concurrent(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator.py" + ) + ) + m.addons.add(sc) + f1, f2 = tutils.tflow(), tutils.tflow() + self.invoke(m, "request", f1) + self.invoke(m, "request", f2) start = time.time() while time.time() - start < 5: if f1.reply.acked and f2.reply.acked: return raise ValueError("Script never acked") - -def test_concurrent_err(): - s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py")) - with tutils.raises("Concurrent decorator not supported for 'start' method"): - s.load() + def test_concurrent_err(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator_err.py" + ) + ) + with m.handlecontext(): + sc.configure(options.Options()) + assert "decorator not supported" in m.event_log[0][1] -- cgit v1.2.3 From b94f5fd361af6255ad4d3c7a88b9a21868736dea Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 16:20:27 +1200 Subject: Convert examples and example tests for new-style scripts Remove the test that just loads all the example scripts for now - it's a very low-value test, and we need to think of something better. --- examples/custom_contentviews.py | 2 +- examples/filt.py | 2 +- examples/flowwriter.py | 2 +- examples/har_extractor.py | 2 +- examples/iframe_injector.py | 2 +- examples/modify_response_body.py | 2 +- examples/proxapp.py | 2 +- examples/stub.py | 6 +- examples/tls_passthrough.py | 2 +- mitmproxy/controller.py | 1 + netlib/utils.py | 7 + test/mitmproxy/script/test_reloader.py | 34 ----- test/mitmproxy/script/test_script.py | 83 ------------ test/mitmproxy/test_examples.py | 234 ++++++++++++++------------------- 14 files changed, 120 insertions(+), 261 deletions(-) delete mode 100644 test/mitmproxy/script/test_reloader.py delete mode 100644 test/mitmproxy/script/test_script.py diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py index 5a63e2a0..b10d936f 100644 --- a/examples/custom_contentviews.py +++ b/examples/custom_contentviews.py @@ -62,7 +62,7 @@ class ViewPigLatin(contentviews.View): pig_view = ViewPigLatin() -def start(): +def configure(options): contentviews.add(pig_view) diff --git a/examples/filt.py b/examples/filt.py index 21744edd..102d1274 100644 --- a/examples/filt.py +++ b/examples/filt.py @@ -6,7 +6,7 @@ from mitmproxy import filt state = {} -def start(): +def configure(options): if len(sys.argv) != 2: raise ValueError("Usage: -s 'filt.py FILTER'") state["filter"] = filt.parse(sys.argv[1]) diff --git a/examples/flowwriter.py b/examples/flowwriter.py index 07c7ca20..d8fbc1f4 100644 --- a/examples/flowwriter.py +++ b/examples/flowwriter.py @@ -6,7 +6,7 @@ from mitmproxy.flow import FlowWriter state = {} -def start(): +def configure(options): if len(sys.argv) != 2: raise ValueError('Usage: -s "flowriter.py filename"') diff --git a/examples/har_extractor.py b/examples/har_extractor.py index 2a69b9af..23deb43a 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -61,7 +61,7 @@ class Context(object): context = Context() -def start(): +def configure(options): """ On start we create a HARLog instance. You will have to adapt this to suit your actual needs of HAR generation. As it will probably be diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py index 70247d31..40934dd3 100644 --- a/examples/iframe_injector.py +++ b/examples/iframe_injector.py @@ -7,7 +7,7 @@ from mitmproxy.models import decoded iframe_url = None -def start(): +def configure(options): if len(sys.argv) != 2: raise ValueError('Usage: -s "iframe_injector.py url"') global iframe_url diff --git a/examples/modify_response_body.py b/examples/modify_response_body.py index 23ad0151..8b6908a4 100644 --- a/examples/modify_response_body.py +++ b/examples/modify_response_body.py @@ -8,7 +8,7 @@ from mitmproxy.models import decoded state = {} -def start(): +def configure(options): if len(sys.argv) != 3: raise ValueError('Usage: -s "modify_response_body.py old new"') # You may want to use Python's argparse for more sophisticated argument diff --git a/examples/proxapp.py b/examples/proxapp.py index 2935b587..095f412a 100644 --- a/examples/proxapp.py +++ b/examples/proxapp.py @@ -16,7 +16,7 @@ def hello_world(): # Register the app using the magic domain "proxapp" on port 80. Requests to # this domain and port combination will now be routed to the WSGI app instance. -def start(): +def configure(options): mitmproxy.ctx.master.apps.add(app, "proxapp", 80) # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. diff --git a/examples/stub.py b/examples/stub.py index 10b34283..614acee2 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -4,11 +4,11 @@ import mitmproxy """ -def start(): +def configure(options): """ - Called once on script startup, before any other events. + Called once on script startup before any other events, and whenever options changes. """ - mitmproxy.ctx.log("start") + mitmproxy.ctx.log("configure") def clientconnect(root_layer): diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py index 20e8f9be..306f55f6 100644 --- a/examples/tls_passthrough.py +++ b/examples/tls_passthrough.py @@ -113,7 +113,7 @@ class TlsFeedback(TlsLayer): tls_strategy = None -def start(): +def configure(options): global tls_strategy if len(sys.argv) == 2: tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 464842b6..bffef58a 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -32,6 +32,7 @@ Events = frozenset([ "error", "log", + "done", "script_change", ]) diff --git a/netlib/utils.py b/netlib/utils.py index 23c16dc3..9eebf22c 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -56,6 +56,13 @@ class Data(object): dirname = os.path.dirname(inspect.getsourcefile(m)) self.dirname = os.path.abspath(dirname) + def push(self, subpath): + """ + Change the data object to a path relative to the module. + """ + self.dirname = os.path.join(self.dirname, subpath) + return self + def path(self, path): """ Returns a path to the package data housed at 'path' under this diff --git a/test/mitmproxy/script/test_reloader.py b/test/mitmproxy/script/test_reloader.py deleted file mode 100644 index e33903b9..00000000 --- a/test/mitmproxy/script/test_reloader.py +++ /dev/null @@ -1,34 +0,0 @@ -import mock -from mitmproxy.script.reloader import watch, unwatch -from test.mitmproxy import tutils -from threading import Event - - -def test_simple(): - with tutils.tmpdir(): - with open("foo.py", "w"): - pass - - script = mock.Mock() - script.path = "foo.py" - - e = Event() - - def _onchange(): - e.set() - - watch(script, _onchange) - with tutils.raises("already observed"): - watch(script, _onchange) - - # Some reloaders don't register a change directly after watching, because they first need to initialize. - # To test if watching works at all, we do repeated writes every 100ms. - for _ in range(100): - with open("foo.py", "a") as f: - f.write(".") - if e.wait(0.1): - break - else: - raise AssertionError("No change detected.") - - unwatch(script) diff --git a/test/mitmproxy/script/test_script.py b/test/mitmproxy/script/test_script.py deleted file mode 100644 index 48fe65c9..00000000 --- a/test/mitmproxy/script/test_script.py +++ /dev/null @@ -1,83 +0,0 @@ -from mitmproxy.script import Script -from mitmproxy.exceptions import ScriptException -from test.mitmproxy import tutils - - -class TestParseCommand: - def test_empty_command(self): - with tutils.raises(ScriptException): - Script.parse_command("") - - with tutils.raises(ScriptException): - Script.parse_command(" ") - - def test_no_script_file(self): - with tutils.raises("not found"): - Script.parse_command("notfound") - - with tutils.tmpdir() as dir: - with tutils.raises("not a file"): - Script.parse_command(dir) - - def test_parse_args(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", []) - assert Script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"]) - assert Script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"]) - - @tutils.skip_not_windows - def test_parse_windows(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", []) - assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar']) - - -def test_simple(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("a.py --var 42") - assert s.path == "a.py" - assert s.ns is None - - s.load() - assert s.ns["var"] == 42 - - s.run("here") - assert s.ns["var"] == 43 - - s.unload() - assert s.ns is None - - with tutils.raises(ScriptException): - s.run("here") - - with Script("a.py --var 42") as s: - s.run("here") - - -def test_script_exception(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("syntaxerr.py") - with tutils.raises(ScriptException): - s.load() - - s = Script("starterr.py") - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py") - s.load() - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py") - with tutils.raises(ScriptException): - s.run("here") - - with tutils.raises(ScriptException): - with Script("reqerr.py") as s: - s.run("request", None) - - s = Script("unloaderr.py") - s.load() - with tutils.raises(ScriptException): - s.unload() diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index bdadcd11..9c8edb29 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,151 +1,119 @@ -import glob import json -import mock -import os -import sys -from contextlib import contextmanager -from mitmproxy import script +import os.path +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options +from mitmproxy import contentviews +from mitmproxy.builtins import script import netlib.utils from netlib import tutils as netutils from netlib.http import Headers -from . import tutils - -example_dir = netlib.utils.Data(__name__).path("../../examples") - - -@contextmanager -def example(command): - command = os.path.join(example_dir, command) - with script.Script(command) as s: - yield s - - -@mock.patch("mitmproxy.ctx.master") -@mock.patch("mitmproxy.ctx.log") -def test_load_scripts(log, master): - scripts = glob.glob("%s/*.py" % example_dir) - - for f in scripts: - if "har_extractor" in f: - continue - if "flowwriter" in f: - f += " -" - if "iframe_injector" in f: - f += " foo" # one argument required - if "filt" in f: - f += " ~a" - if "modify_response_body" in f: - f += " foo bar" # two arguments required - - s = script.Script(f) - try: - s.load() - except Exception as v: - if "ImportError" not in str(v): - raise - else: - s.unload() - - -def test_add_header(): - flow = tutils.tflow(resp=netutils.tresp()) - with example("add_header.py") as ex: - ex.run("response", flow) - assert flow.response.headers["newheader"] == "foo" - - -@mock.patch("mitmproxy.contentviews.remove") -@mock.patch("mitmproxy.contentviews.add") -def test_custom_contentviews(add, remove): - with example("custom_contentviews.py"): - assert add.called - pig = add.call_args[0][0] - _, fmt = pig(b"test!") - assert any(b'esttay!' in val[0][1] for val in fmt) - assert not pig(b"gobbledygook") - assert remove.called - - -def test_iframe_injector(): - with tutils.raises(script.ScriptException): - with example("iframe_injector.py"): - pass - - flow = tutils.tflow(resp=netutils.tresp(content=b"mitmproxy")) - with example("iframe_injector.py http://example.org/evil_iframe") as ex: - ex.run("response", flow) - content = flow.response.content - assert b'iframe' in content and b'evil_iframe' in content - - -def test_modify_form(): - form_header = Headers(content_type="application/x-www-form-urlencoded") - flow = tutils.tflow(req=netutils.treq(headers=form_header)) - with example("modify_form.py") as ex: - ex.run("request", flow) - assert flow.request.urlencoded_form[b"mitmproxy"] == b"rocks" - - flow.request.headers["content-type"] = "" - ex.run("request", flow) - assert list(flow.request.urlencoded_form.items()) == [(b"foo", b"bar")] +from . import tutils, mastertest +example_dir = netlib.utils.Data(__name__).push("../../examples") -def test_modify_querystring(): - flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term")) - with example("modify_querystring.py") as ex: - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" - flow.request.path = "/" - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" +class ScriptError(Exception): + pass -def test_modify_response_body(): - with tutils.raises(script.ScriptException): - with example("modify_response_body.py"): - assert True +class RaiseMaster(master.FlowMaster): + def add_event(self, e, level): + if level in ("warn", "error"): + raise ScriptError(e) - flow = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) - with example("modify_response_body.py mitmproxy rocks") as ex: - assert ex.ns["state"]["old"] == b"mitmproxy" and ex.ns["state"]["new"] == b"rocks" - ex.run("response", flow) - assert flow.response.content == b"I <3 rocks" +def tscript(cmd, args=""): + cmd = example_dir.path(cmd) + " " + args + m = RaiseMaster(options.Options(), None, state.State()) + sc = script.Script(cmd) + m.addons.add(sc) + return m, sc -def test_redirect_requests(): - flow = tutils.tflow(req=netutils.treq(host=b"example.org")) - with example("redirect_requests.py") as ex: - ex.run("request", flow) - assert flow.request.host == "mitmproxy.org" +class TestScripts(mastertest.MasterTest): + def test_add_header(self): + m, _ = tscript("add_header.py") + f = tutils.tflow(resp=netutils.tresp()) + self.invoke(m, "response", f) + assert f.response.headers["newheader"] == "foo" -@mock.patch("mitmproxy.ctx.log") -def test_har_extractor(log): - if sys.version_info >= (3, 0): - with tutils.raises("does not work on Python 3"): - with example("har_extractor.py -"): - pass - return + def test_custom_contentviews(self): + m, sc = tscript("custom_contentviews.py") + pig = contentviews.get("pig_latin_HTML") + _, fmt = pig("test!") + assert any('esttay!' in val[0][1] for val in fmt) + assert not pig("gobbledygook") - with tutils.raises(script.ScriptException): - with example("har_extractor.py"): - pass + def test_iframe_injector(self): + with tutils.raises(ScriptError): + tscript("iframe_injector.py") - times = dict( - timestamp_start=746203272, - timestamp_end=746203272, - ) - - flow = tutils.tflow( - req=netutils.treq(**times), - resp=netutils.tresp(**times) - ) - - with example("har_extractor.py -") as ex: - ex.run("response", flow) - - with open(tutils.test_data.path("data/har_extractor.har")) as fp: + m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") + flow = tutils.tflow(resp=netutils.tresp(content="mitmproxy")) + self.invoke(m, "response", flow) + content = flow.response.content + assert 'iframe' in content and 'evil_iframe' in content + + def test_modify_form(self): + m, sc = tscript("modify_form.py") + + form_header = Headers(content_type="application/x-www-form-urlencoded") + f = tutils.tflow(req=netutils.treq(headers=form_header)) + self.invoke(m, "request", f) + + assert f.request.urlencoded_form["mitmproxy"] == "rocks" + + f.request.headers["content-type"] = "" + self.invoke(m, "request", f) + assert list(f.request.urlencoded_form.items()) == [("foo", "bar")] + + def test_modify_querystring(self): + m, sc = tscript("modify_querystring.py") + f = tutils.tflow(req=netutils.treq(path="/search?q=term")) + + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + f.request.path = "/" + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + def test_modify_response_body(self): + with tutils.raises(ScriptError): + tscript("modify_response_body.py") + + m, sc = tscript("modify_response_body.py", "mitmproxy rocks") + f = tutils.tflow(resp=netutils.tresp(content="I <3 mitmproxy")) + self.invoke(m, "response", f) + assert f.response.content == "I <3 rocks" + + def test_redirect_requests(self): + m, sc = tscript("redirect_requests.py") + f = tutils.tflow(req=netutils.treq(host="example.org")) + self.invoke(m, "request", f) + assert f.request.host == "mitmproxy.org" + + def test_har_extractor(self): + with tutils.raises(ScriptError): + tscript("har_extractor.py") + + with tutils.tmpdir() as tdir: + times = dict( + timestamp_start=746203272, + timestamp_end=746203272, + ) + + path = os.path.join(tdir, "file") + m, sc = tscript("har_extractor.py", path) + f = tutils.tflow( + req=netutils.treq(**times), + resp=netutils.tresp(**times) + ) + self.invoke(m, "response", f) + m.addons.remove(sc) + + fp = open(path, "rb") test_data = json.load(fp) - assert json.loads(ex.ns["context"].HARLog.json()) == test_data["test_response"] + assert len(test_data["log"]["pages"]) == 1 -- cgit v1.2.3 From deffed2196a8d595624998b9fcc8fa4016b41808 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 17:19:33 +1200 Subject: Script cleanup: editing in console, Python3 compatibility fixes --- mitmproxy/console/grideditor.py | 11 ++++++----- mitmproxy/console/master.py | 32 +------------------------------- mitmproxy/console/options.py | 16 +++++++++------- mitmproxy/console/statusbar.py | 9 ++++----- test/mitmproxy/script/test_concurrent.py | 3 +-- test/mitmproxy/test_examples.py | 24 +++++++++++++++--------- test/mitmproxy/test_server.py | 4 ++-- 7 files changed, 38 insertions(+), 61 deletions(-) diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py index 9fa51ccb..f304de57 100644 --- a/mitmproxy/console/grideditor.py +++ b/mitmproxy/console/grideditor.py @@ -6,11 +6,12 @@ import re import urwid +from mitmproxy import exceptions from mitmproxy import filt -from mitmproxy import script -from mitmproxy import utils +from mitmproxy.builtins import script from mitmproxy.console import common from mitmproxy.console import signals +from netlib import strutils from netlib.http import cookies from netlib.http import user_agents @@ -55,7 +56,7 @@ class TextColumn: o = editor.walker.get_current_value() if o is not None: n = editor.master.spawn_editor(o.encode("string-escape")) - n = utils.clean_hanging_newline(n) + n = strutils.clean_hanging_newline(n) editor.walker.set_current_value(n, False) editor.walker._modified() elif key in ["enter"]: @@ -643,8 +644,8 @@ class ScriptEditor(GridEditor): def is_error(self, col, val): try: - script.Script.parse_command(val) - except script.ScriptException as e: + script.parse_command(val) + except exceptions.AddonError as e: return str(e) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index bc373a2b..64bd9f0a 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -248,23 +248,6 @@ class ConsoleMaster(flow.FlowMaster): if options.server_replay: self.server_playback_path(options.server_replay) - if options.scripts: - for i in options.scripts: - try: - self.load_script(i) - except exceptions.ScriptException as e: - print("Script load error: {}".format(e), file=sys.stderr) - sys.exit(1) - - if options.outfile: - err = self.start_stream_to_path( - options.outfile[0], - options.outfile[1] - ) - if err: - print("Stream file error: {}".format(err), file=sys.stderr) - sys.exit(1) - self.view_stack = [] if options.app: @@ -685,20 +668,7 @@ class ConsoleMaster(flow.FlowMaster): self.refresh_focus() def edit_scripts(self, scripts): - commands = [x[0] for x in scripts] # remove outer array - if commands == [s.command for s in self.scripts]: - return - - self.unload_scripts() - for command in commands: - try: - self.load_script(command) - except exceptions.ScriptException as e: - signals.status_message.send( - message='Error loading "{}".'.format(command) - ) - signals.add_event('Error loading "{}":\n{}'.format(command, e), "error") - signals.update_settings.send(self) + self.options.scripts = [x[0] for x in scripts] def stop_client_playback_prompt(self, a): if a != "n": diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index d363ba74..d8824b05 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -54,7 +54,7 @@ class Options(urwid.WidgetWrap): select.Option( "Scripts", "S", - lambda: master.scripts, + lambda: master.options.scripts, self.scripts ), @@ -160,12 +160,14 @@ class Options(urwid.WidgetWrap): self.master.replacehooks.clear() self.master.set_ignore_filter([]) self.master.set_tcp_filter([]) - self.master.scripts = [] - self.master.options.anticache = False - self.master.options.anticomp = False - self.master.options.stickyauth = None - self.master.options.stickycookie = None + self.master.options.update( + scripts = [], + anticache = False, + anticomp = False, + stickyauth = None, + stickycookie = None + ) self.master.state.default_body_view = contentviews.get("Auto") @@ -234,7 +236,7 @@ class Options(urwid.WidgetWrap): self.master.view_grideditor( grideditor.ScriptEditor( self.master, - [[i.command] for i in self.master.scripts], + [[i] for i in self.master.options.scripts], self.master.edit_scripts ) ) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index fc41869c..e7a700a6 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -218,14 +218,13 @@ class StatusBar(urwid.WidgetWrap): dst.address.host, dst.address.port )) - if self.master.scripts: + if self.master.options.scripts: r.append("[") r.append(("heading_key", "s")) - r.append("cripts:%s]" % len(self.master.scripts)) - # r.append("[lt:%0.3f]"%self.master.looptime) + r.append("cripts:%s]" % len(self.master.options.scripts)) - if self.master.stream: - r.append("[W:%s]" % self.master.stream_path) + if self.master.options.outfile: + r.append("[W:%s]" % self.master.outfile[0]) return r diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index d5243bcb..92d1153b 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,11 +1,10 @@ -from test.mitmproxy import tutils +from test.mitmproxy import tutils, mastertest from mitmproxy import controller from mitmproxy.builtins import script from mitmproxy import options from mitmproxy.flow import master from mitmproxy.flow import state import time -from .. import mastertest, tutils class Thing: diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 9c8edb29..ef97219c 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,5 +1,6 @@ import json +import sys import os.path from mitmproxy.flow import master from mitmproxy.flow import state @@ -42,19 +43,19 @@ class TestScripts(mastertest.MasterTest): def test_custom_contentviews(self): m, sc = tscript("custom_contentviews.py") pig = contentviews.get("pig_latin_HTML") - _, fmt = pig("test!") - assert any('esttay!' in val[0][1] for val in fmt) - assert not pig("gobbledygook") + _, fmt = pig(b"test!") + assert any(b'esttay!' in val[0][1] for val in fmt) + assert not pig(b"gobbledygook") def test_iframe_injector(self): with tutils.raises(ScriptError): tscript("iframe_injector.py") m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") - flow = tutils.tflow(resp=netutils.tresp(content="mitmproxy")) + flow = tutils.tflow(resp=netutils.tresp(content=b"mitmproxy")) self.invoke(m, "response", flow) content = flow.response.content - assert 'iframe' in content and 'evil_iframe' in content + assert b'iframe' in content and b'evil_iframe' in content def test_modify_form(self): m, sc = tscript("modify_form.py") @@ -63,11 +64,11 @@ class TestScripts(mastertest.MasterTest): f = tutils.tflow(req=netutils.treq(headers=form_header)) self.invoke(m, "request", f) - assert f.request.urlencoded_form["mitmproxy"] == "rocks" + assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks" f.request.headers["content-type"] = "" self.invoke(m, "request", f) - assert list(f.request.urlencoded_form.items()) == [("foo", "bar")] + assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")] def test_modify_querystring(self): m, sc = tscript("modify_querystring.py") @@ -85,9 +86,9 @@ class TestScripts(mastertest.MasterTest): tscript("modify_response_body.py") m, sc = tscript("modify_response_body.py", "mitmproxy rocks") - f = tutils.tflow(resp=netutils.tresp(content="I <3 mitmproxy")) + f = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) self.invoke(m, "response", f) - assert f.response.content == "I <3 rocks" + assert f.response.content == b"I <3 rocks" def test_redirect_requests(self): m, sc = tscript("redirect_requests.py") @@ -96,6 +97,11 @@ class TestScripts(mastertest.MasterTest): assert f.request.host == "mitmproxy.org" def test_har_extractor(self): + if sys.version_info >= (3, 0): + with tutils.raises("does not work on Python 3"): + tscript("har_extractor.py") + return + with tutils.raises(ScriptError): tscript("har_extractor.py") diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index b1ca6910..a5196dae 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -293,7 +293,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): ) self.master.addons.add(s) d = self.pathod('200:b"foo"') - assert d.content == "bar" + assert d.content == b"bar" self.master.addons.remove(s) @@ -523,7 +523,7 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() - assert d.content == "bar" + assert d.content == b"bar" self.master.addons.remove(s) -- cgit v1.2.3 From 5b2d1c044a0683444f117d8085e29bb613dbbf9d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 17:22:22 +1200 Subject: Tighten the tick loop In the past, we consumed from the event queue until we were idle for a certain amount of time (0.1s). This would cause hangs in interactive tools when there was a stream of events, hurting responsiveness. We now wait for a maximum of 0.1s before triggering the tick loop, will be able to reduce this further down the track. --- mitmproxy/controller.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index bffef58a..72f8e001 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -110,24 +110,21 @@ class Master(object): def tick(self, timeout): changed = False try: - # This endless loop runs until the 'Queue.Empty' - # exception is thrown. - while True: - mtype, obj = self.event_queue.get(timeout=timeout) - if mtype not in Events: - raise exceptions.ControlException("Unknown event %s" % repr(mtype)) - handle_func = getattr(self, mtype) - if not callable(handle_func): - raise exceptions.ControlException("Handler %s not callable" % mtype) - if not handle_func.__dict__.get("__handler"): - raise exceptions.ControlException( - "Handler function %s is not decorated with controller.handler" % ( - handle_func - ) + mtype, obj = self.event_queue.get(timeout=timeout) + if mtype not in Events: + raise exceptions.ControlException("Unknown event %s" % repr(mtype)) + handle_func = getattr(self, mtype) + if not callable(handle_func): + raise exceptions.ControlException("Handler %s not callable" % mtype) + if not handle_func.__dict__.get("__handler"): + raise exceptions.ControlException( + "Handler function %s is not decorated with controller.handler" % ( + handle_func ) - handle_func(obj) - self.event_queue.task_done() - changed = True + ) + handle_func(obj) + self.event_queue.task_done() + changed = True except queue.Empty: pass return changed -- cgit v1.2.3 From a4127fb6d5f026c015525bab7993bf6f33e16f93 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 18:46:07 +1200 Subject: Add a tick event for addons and scripts, and use it for race-free reload --- mitmproxy/builtins/script.py | 34 ++++++++++++++++++++-------------- mitmproxy/controller.py | 3 +++ test/mitmproxy/builtins/test_script.py | 1 + 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index 015adef9..34801ff7 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -4,8 +4,8 @@ import contextlib import os import shlex import sys +import threading import traceback -import copy from mitmproxy import exceptions from mitmproxy import controller @@ -76,15 +76,14 @@ def load_script(path, args): class ReloadHandler(watchdog.events.FileSystemEventHandler): - def __init__(self, callback, master, options): + def __init__(self, callback): self.callback = callback - self.master, self.options = master, options def on_modified(self, event): - self.callback(self.master, self.options) + self.callback() def on_created(self, event): - self.callback(self.master, self.options) + self.callback() class Script: @@ -99,7 +98,10 @@ class Script: self.ns = None self.observer = None - for i in controller.Events: + self.last_options = None + self.should_reload = threading.Event() + + for i in controller.Events - set(["tick"]): def mkprox(): evt = i @@ -117,21 +119,25 @@ class Script: with scriptenv(self.path, self.args): func(*args, **kwargs) - def reload(self, master, options): - with master.handlecontext(): + def reload(self): + self.should_reload.set() + + def tick(self): + if self.should_reload.is_set(): + self.should_reload.clear() self.ns = None - self.configure(options) + ctx.log.info("Reloading script: %s" % self.name) + self.configure(self.last_options) + else: + self.run("tick") def configure(self, options): + self.last_options = options if not self.observer: self.observer = Observer() # Bind the handler to the real underlying master object self.observer.schedule( - ReloadHandler( - self.reload, - ctx.master, - copy.copy(options), - ), + ReloadHandler(self.reload), os.path.dirname(self.path) or "." ) self.observer.start() diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 72f8e001..d3ae1baa 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -33,6 +33,7 @@ Events = frozenset([ "error", "log", "done", + "tick", "script_change", ]) @@ -108,6 +109,8 @@ class Master(object): self.shutdown() def tick(self, timeout): + with self.handlecontext(): + self.addons("tick") changed = False try: mtype, obj = self.event_queue.get(timeout=timeout) diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index 2447c8ea..394c3f38 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -78,6 +78,7 @@ class TestScript(mastertest.MasterTest): for _ in range(100): with open("foo.py", "a") as f: f.write(".") + m.addons.invoke_with_context(sc, "tick") time.sleep(0.1) if m.event_log: return -- cgit v1.2.3 From 698af945a49ef34e909f1f5dc1f12552f9e3680b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 19:58:12 +1200 Subject: Adjust for new options scheme --- test/mitmproxy/builtins/test_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index 394c3f38..7c9787f8 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -4,7 +4,7 @@ from mitmproxy.builtins import script from mitmproxy import exceptions from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy import options +from mitmproxy.flow import options from .. import tutils, mastertest -- cgit v1.2.3 From e051928f26afbc58c01d7854b62f8bfb0d4ed5ee Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 20:24:56 +1200 Subject: Add missing mitmweb option --- mitmproxy/web/master.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index 75cc7746..adfc23c4 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -94,6 +94,7 @@ class WebState(flow.State): class Options(flow.options.Options): def __init__( self, + intercept=False, # type: bool wdebug=bool, # type: bool wport=8081, # type: int wiface="127.0.0.1", # type: str @@ -108,6 +109,7 @@ class Options(flow.options.Options): self.wauthenticator = wauthenticator self.wsingleuser = wsingleuser self.whtpasswd = whtpasswd + self.intercept = intercept super(Options, self).__init__(**kwargs) # TODO: This doesn't belong here. -- cgit v1.2.3 From 8fee5db675744a6cc9a3462b7d1261e0b940545c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Jul 2016 21:33:53 +1200 Subject: Always use PollingObserver --- mitmproxy/builtins/script.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index 34801ff7..f6cf093f 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -13,12 +13,7 @@ from mitmproxy import ctx import watchdog.events -# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths. -# We use the PollingObserver instead. -if sys.platform == 'darwin': # pragma: no cover - from watchdog.observers.polling import PollingObserver as Observer -else: - from watchdog.observers import Observer +from watchdog.observers import polling def parse_command(command): @@ -134,7 +129,7 @@ class Script: def configure(self, options): self.last_options = options if not self.observer: - self.observer = Observer() + self.observer = polling.PollingObserver() # Bind the handler to the real underlying master object self.observer.schedule( ReloadHandler(self.reload), -- cgit v1.2.3 From fcc1416ffd82a3497bf17323b4bb467e7e4435f4 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 10:33:51 +1200 Subject: Fix windows parse_args test --- test/mitmproxy/builtins/test_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index 7c9787f8..5747912d 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -35,7 +35,7 @@ class TestParseCommand: def test_parse_windows(self): with tutils.chdir(tutils.test_data.dirname): assert script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", []) - assert script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", 'foo \\ bar', []) + assert script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar']) def test_load_script(): -- cgit v1.2.3 From 4ba7ce50c6aa7b96325e201f65747a3a6ace1a7a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 13:22:20 +1200 Subject: Add .start for addons and scripts Also improve error messages, fix various unit tests --- examples/custom_contentviews.py | 2 +- examples/filt.py | 2 +- examples/flowwriter.py | 2 +- examples/har_extractor.py | 2 +- examples/iframe_injector.py | 2 +- examples/modify_response_body.py | 2 +- examples/proxapp.py | 2 +- examples/stub.py | 9 +++++++++ examples/tls_passthrough.py | 2 +- mitmproxy/addons.py | 1 + mitmproxy/builtins/script.py | 29 +++++++++++++++++++---------- mitmproxy/controller.py | 3 +++ mitmproxy/script/concurrent.py | 2 +- test/mitmproxy/builtins/test_script.py | 7 +++++-- 14 files changed, 46 insertions(+), 21 deletions(-) diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py index b10d936f..5a63e2a0 100644 --- a/examples/custom_contentviews.py +++ b/examples/custom_contentviews.py @@ -62,7 +62,7 @@ class ViewPigLatin(contentviews.View): pig_view = ViewPigLatin() -def configure(options): +def start(): contentviews.add(pig_view) diff --git a/examples/filt.py b/examples/filt.py index 102d1274..21744edd 100644 --- a/examples/filt.py +++ b/examples/filt.py @@ -6,7 +6,7 @@ from mitmproxy import filt state = {} -def configure(options): +def start(): if len(sys.argv) != 2: raise ValueError("Usage: -s 'filt.py FILTER'") state["filter"] = filt.parse(sys.argv[1]) diff --git a/examples/flowwriter.py b/examples/flowwriter.py index d8fbc1f4..07c7ca20 100644 --- a/examples/flowwriter.py +++ b/examples/flowwriter.py @@ -6,7 +6,7 @@ from mitmproxy.flow import FlowWriter state = {} -def configure(options): +def start(): if len(sys.argv) != 2: raise ValueError('Usage: -s "flowriter.py filename"') diff --git a/examples/har_extractor.py b/examples/har_extractor.py index 23deb43a..2a69b9af 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -61,7 +61,7 @@ class Context(object): context = Context() -def configure(options): +def start(): """ On start we create a HARLog instance. You will have to adapt this to suit your actual needs of HAR generation. As it will probably be diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py index 40934dd3..70247d31 100644 --- a/examples/iframe_injector.py +++ b/examples/iframe_injector.py @@ -7,7 +7,7 @@ from mitmproxy.models import decoded iframe_url = None -def configure(options): +def start(): if len(sys.argv) != 2: raise ValueError('Usage: -s "iframe_injector.py url"') global iframe_url diff --git a/examples/modify_response_body.py b/examples/modify_response_body.py index 8b6908a4..23ad0151 100644 --- a/examples/modify_response_body.py +++ b/examples/modify_response_body.py @@ -8,7 +8,7 @@ from mitmproxy.models import decoded state = {} -def configure(options): +def start(): if len(sys.argv) != 3: raise ValueError('Usage: -s "modify_response_body.py old new"') # You may want to use Python's argparse for more sophisticated argument diff --git a/examples/proxapp.py b/examples/proxapp.py index 095f412a..2935b587 100644 --- a/examples/proxapp.py +++ b/examples/proxapp.py @@ -16,7 +16,7 @@ def hello_world(): # Register the app using the magic domain "proxapp" on port 80. Requests to # this domain and port combination will now be routed to the WSGI app instance. -def configure(options): +def start(): mitmproxy.ctx.master.apps.add(app, "proxapp", 80) # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. diff --git a/examples/stub.py b/examples/stub.py index 614acee2..7de4012a 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -4,6 +4,15 @@ import mitmproxy """ + +def start(): + """ + Called once on script startup before any other events + """ + mitmproxy.ctx.log("start") + + + def configure(options): """ Called once on script startup before any other events, and whenever options changes. diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py index 306f55f6..20e8f9be 100644 --- a/examples/tls_passthrough.py +++ b/examples/tls_passthrough.py @@ -113,7 +113,7 @@ class TlsFeedback(TlsLayer): tls_strategy = None -def configure(options): +def start(): global tls_strategy if len(sys.argv) == 2: tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) diff --git a/mitmproxy/addons.py b/mitmproxy/addons.py index 7ac65a09..c779aaf8 100644 --- a/mitmproxy/addons.py +++ b/mitmproxy/addons.py @@ -21,6 +21,7 @@ class Addons(object): def add(self, *addons): self.chain.extend(addons) for i in addons: + self.invoke_with_context(i, "start") self.invoke_with_context(i, "configure", self.master.options) def remove(self, addon): diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index f6cf093f..bcb756f6 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -51,22 +51,29 @@ def scriptenv(path, args): _, _, tb = sys.exc_info() scriptdir = os.path.dirname(os.path.abspath(path)) for i, s in enumerate(reversed(traceback.extract_tb(tb))): + tb = tb.tb_next if not os.path.abspath(s[0]).startswith(scriptdir): break - else: - tb = tb.tb_next - ctx.log.warn("".join(traceback.format_tb(tb))) + ctx.log.error("Script error: %s" % "".join(traceback.format_tb(tb))) finally: sys.argv = oldargs sys.path.pop() def load_script(path, args): + with open(path, "rb") as f: + try: + code = compile(f.read(), path, 'exec') + except SyntaxError as e: + ctx.log.error( + "Script error: %s line %s: %s" % ( + e.filename, e.lineno, e.msg + ) + ) + return ns = {'__file__': os.path.abspath(path)} with scriptenv(path, args): - with open(path, "rb") as f: - code = compile(f.read(), path, 'exec') - exec(code, ns, ns) + exec(code, ns, ns) return ns @@ -96,7 +103,7 @@ class Script: self.last_options = None self.should_reload = threading.Event() - for i in controller.Events - set(["tick"]): + for i in controller.Events - set(["start", "configure", "tick"]): def mkprox(): evt = i @@ -120,12 +127,16 @@ class Script: def tick(self): if self.should_reload.is_set(): self.should_reload.clear() - self.ns = None ctx.log.info("Reloading script: %s" % self.name) + self.ns = load_script(self.path, self.args) self.configure(self.last_options) else: self.run("tick") + def start(self): + self.ns = load_script(self.path, self.args) + self.run("start") + def configure(self, options): self.last_options = options if not self.observer: @@ -136,8 +147,6 @@ class Script: os.path.dirname(self.path) or "." ) self.observer.start() - if not self.ns: - self.ns = load_script(self.path, self.args) self.run("configure", options) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index d3ae1baa..503cdcd3 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -32,6 +32,9 @@ Events = frozenset([ "error", "log", + + "start", + "configure", "done", "tick", diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index 010a5fa0..0cc0514e 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -13,7 +13,7 @@ class ScriptThread(basethread.BaseThread): def concurrent(fn): - if fn.__name__ not in controller.Events: + if fn.__name__ not in controller.Events - set(["start", "configure", "tick"]): raise NotImplementedError( "Concurrent decorator not supported for '%s' method." % fn.__name__ ) diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index 5747912d..2c2568ed 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -57,7 +57,10 @@ class TestScript(mastertest.MasterTest): ) ) m.addons.add(sc) - assert sc.ns["call_log"] == [("configure", (options.Options(),), {})] + assert sc.ns["call_log"] == [ + ("start", (), {}), + ("configure", (options.Options(),), {}) + ] sc.ns["call_log"] = [] f = tutils.tflow(resp=True) @@ -93,7 +96,7 @@ class TestScript(mastertest.MasterTest): m.addons.add(sc) f = tutils.tflow(resp=True) self.invoke(m, "request", f) - assert m.event_log[0][0] == "warn" + assert m.event_log[0][0] == "error" def test_duplicate_flow(self): s = state.State() -- cgit v1.2.3 From 92a5076bd01b71e8ed709901d63de9ee37385c3b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 14:59:48 +1200 Subject: Adjust concurrent tests for start method --- examples/stub.py | 2 -- test/mitmproxy/script/test_concurrent.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/stub.py b/examples/stub.py index 7de4012a..e5b4a39a 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -4,7 +4,6 @@ import mitmproxy """ - def start(): """ Called once on script startup before any other events @@ -12,7 +11,6 @@ def start(): mitmproxy.ctx.log("start") - def configure(options): """ Called once on script startup before any other events, and whenever options changes. diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 92d1153b..080746e8 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -42,5 +42,5 @@ class TestConcurrent(mastertest.MasterTest): ) ) with m.handlecontext(): - sc.configure(options.Options()) + sc.start() assert "decorator not supported" in m.event_log[0][1] -- cgit v1.2.3 From 917d51bd22ea75408bcf461b09b6cf78c44e1162 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 15:01:35 +1200 Subject: Fix HAR extractor Thanks to @mhils --- examples/har_extractor.py | 8 +++++--- test/mitmproxy/test_examples.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/har_extractor.py b/examples/har_extractor.py index 2a69b9af..90412ec0 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -2,7 +2,7 @@ This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser to generate a HAR log object. """ -import mitmproxy +import mitmproxy.ctx import six import sys import pytz @@ -221,9 +221,11 @@ def done(): if context.dump_file == '-': mitmproxy.ctx.log(pprint.pformat(json.loads(json_dump))) elif context.dump_file.endswith('.zhar'): - file(context.dump_file, "w").write(compressed_json_dump) + with open(context.dump_file, "wb") as f: + f.write(compressed_json_dump) else: - file(context.dump_file, "w").write(json_dump) + with open(context.dump_file, "wb") as f: + f.write(json_dump) mitmproxy.ctx.log( "HAR log finished with %s bytes (%s bytes compressed)" % ( len(json_dump), len(compressed_json_dump) diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index ef97219c..f8646336 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,5 +1,6 @@ import json +import six import sys import os.path from mitmproxy.flow import master @@ -112,7 +113,7 @@ class TestScripts(mastertest.MasterTest): ) path = os.path.join(tdir, "file") - m, sc = tscript("har_extractor.py", path) + m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path)) f = tutils.tflow( req=netutils.treq(**times), resp=netutils.tresp(**times) @@ -120,6 +121,6 @@ class TestScripts(mastertest.MasterTest): self.invoke(m, "response", f) m.addons.remove(sc) - fp = open(path, "rb") - test_data = json.load(fp) + with open(path, "rb") as f: + test_data = json.load(f) assert len(test_data["log"]["pages"]) == 1 -- cgit v1.2.3 From c7d0850d8f697915b183f4fafd5ede7df2245569 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 16:35:24 +1200 Subject: Script cleanups - Preserve script order on config change - Prohibit script duplicates (i.e. identical script + args) - Various cleanups and tweaks --- mitmproxy/builtins/script.py | 47 ++++++++++++++------- test/mitmproxy/builtins/test_script.py | 62 ++++++++++++++++++++++++++-- test/mitmproxy/data/addonscripts/recorder.py | 9 +++- 3 files changed, 100 insertions(+), 18 deletions(-) diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index bcb756f6..14f7dd5f 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -99,23 +99,25 @@ class Script: self.path, self.args = parse_command(command) self.ns = None self.observer = None + self.dead = False self.last_options = None self.should_reload = threading.Event() - for i in controller.Events - set(["start", "configure", "tick"]): - def mkprox(): - evt = i + for i in controller.Events: + if not hasattr(self, i): + def mkprox(): + evt = i - def prox(*args, **kwargs): - self.run(evt, *args, **kwargs) - return prox - setattr(self, i, mkprox()) + def prox(*args, **kwargs): + self.run(evt, *args, **kwargs) + return prox + setattr(self, i, mkprox()) def run(self, name, *args, **kwargs): # It's possible for ns to be un-initialised if we failed during # configure - if self.ns is not None: + if self.ns is not None and not self.dead: func = self.ns.get(name) if func: with scriptenv(self.path, self.args): @@ -149,18 +151,35 @@ class Script: self.observer.start() self.run("configure", options) + def done(self): + self.run("done") + self.dead = True + class ScriptLoader(): """ An addon that manages loading scripts from options. """ def configure(self, options): - for s in options.scripts or []: - if not ctx.master.addons.has_addon(s): + for s in options.scripts: + if options.scripts.count(s) > 1: + raise exceptions.OptionsError("Duplicate script: %s" % s) + + for a in ctx.master.addons.chain[:]: + if isinstance(a, Script) and a.name not in options.scripts: + ctx.log.info("Un-loading script: %s" % a.name) + ctx.master.addons.remove(a) + + current = {} + for a in ctx.master.addons.chain[:]: + if isinstance(a, Script): + current[a.name] = a + ctx.master.addons.chain.remove(a) + + for s in options.scripts: + if s in current: + ctx.master.addons.chain.append(current[s]) + else: ctx.log.info("Loading script: %s" % s) sc = Script(s) ctx.master.addons.add(sc) - for a in ctx.master.addons.chain: - if isinstance(a, Script): - if a.name not in options.scripts or []: - ctx.master.addons.remove(a) diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index 2c2568ed..da60b64c 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -58,8 +58,8 @@ class TestScript(mastertest.MasterTest): ) m.addons.add(sc) assert sc.ns["call_log"] == [ - ("start", (), {}), - ("configure", (options.Options(),), {}) + ("solo", "start", (), {}), + ("solo", "configure", (options.Options(),), {}) ] sc.ns["call_log"] = [] @@ -67,7 +67,7 @@ class TestScript(mastertest.MasterTest): self.invoke(m, "request", f) recf = sc.ns["call_log"][0] - assert recf[0] == "request" + assert recf[1] == "request" def test_reload(self): s = state.State() @@ -129,3 +129,59 @@ class TestScriptLoader(mastertest.MasterTest): assert len(m.addons) == 2 o.update(scripts = []) assert len(m.addons) == 1 + + def test_dupes(self): + s = state.State() + o = options.Options(scripts=["one", "one"]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + tutils.raises(exceptions.OptionsError, m.addons.add, sc) + + def test_order(self): + rec = tutils.test_data.path("data/addonscripts/recorder.py") + + s = state.State() + o = options.Options( + scripts = [ + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + "%s %s" % (rec, "c"), + ] + ) + m = mastertest.RecordingMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'a start'), ('debug', 'a configure'), + ('debug', 'b start'), ('debug', 'b configure'), + ('debug', 'c start'), ('debug', 'c configure') + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "c"), + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c configure'), + ('debug', 'a configure'), + ('debug', 'b configure'), + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "x"), + "%s %s" % (rec, "a"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c done'), + ('debug', 'b done'), + ('debug', 'x start'), + ('debug', 'x configure'), + ('debug', 'a configure'), + ] diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py index 728203e3..b6ac8d89 100644 --- a/test/mitmproxy/data/addonscripts/recorder.py +++ b/test/mitmproxy/data/addonscripts/recorder.py @@ -1,8 +1,14 @@ from mitmproxy import controller from mitmproxy import ctx +import sys call_log = [] +if len(sys.argv) > 1: + name = sys.argv[1] +else: + name = "solo" + # Keep a log of all possible event calls evts = list(controller.Events) + ["configure"] for i in evts: @@ -10,9 +16,10 @@ for i in evts: evt = i def prox(*args, **kwargs): - lg = (evt, args, kwargs) + lg = (name, evt, args, kwargs) if evt != "log": ctx.log.info(str(lg)) call_log.append(lg) + ctx.log.debug("%s %s" % (name, evt)) return prox globals()[i] = mkprox() -- cgit v1.2.3 From 60d2e14cb9617d98d24cb3278f2f7245bef91a29 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Jul 2016 00:07:32 -0700 Subject: fix #1358 --- mitmproxy/console/flowlist.py | 14 ++++++-------- mitmproxy/console/statusbar.py | 8 +++++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index 8c20c4b6..2f167a4d 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -317,11 +317,9 @@ class FlowListWalker(urwid.ListWalker): class FlowListBox(urwid.ListBox): def __init__(self, master): + # type: (mitmproxy.console.master.ConsoleMaster) -> None self.master = master - urwid.ListBox.__init__( - self, - FlowListWalker(master, master.state) - ) + super(FlowListBox, self).__init__(FlowListWalker(master, master.state)) def get_method_raw(self, k): if k: @@ -395,13 +393,13 @@ class FlowListBox(urwid.ListBox): elif key == "F": self.master.toggle_follow_flows() elif key == "W": - if self.master.stream: - self.master.stop_stream() + if self.master.options.outfile: + self.master.options.outfile = None else: signals.status_prompt_path.send( self, - prompt = "Stream flows to", - callback = self.master.start_stream_to_path + prompt="Stream flows to", + callback= lambda path: self.master.options.update(outfile=(path, "ab")) ) else: return urwid.ListBox.keypress(self, size, key) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index e7a700a6..040ea26d 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -116,10 +116,12 @@ class ActionBar(urwid.WidgetWrap): class StatusBar(urwid.WidgetWrap): def __init__(self, master, helptext): - self.master, self.helptext = master, helptext + # type: (mitmproxy.console.master.ConsoleMaster, object) -> None + self.master = master + self.helptext = helptext self.ab = ActionBar() self.ib = urwid.WidgetWrap(urwid.Text("")) - self._w = urwid.Pile([self.ib, self.ab]) + super(StatusBar, self).__init__(urwid.Pile([self.ib, self.ab])) signals.update_settings.connect(self.sig_update_settings) signals.flowlist_change.connect(self.sig_update_settings) master.options.changed.connect(self.sig_update_settings) @@ -224,7 +226,7 @@ class StatusBar(urwid.WidgetWrap): r.append("cripts:%s]" % len(self.master.options.scripts)) if self.master.options.outfile: - r.append("[W:%s]" % self.master.outfile[0]) + r.append("[W:%s]" % self.master.options.outfile[0]) return r -- cgit v1.2.3 From c4fb489a96c58932609cced96d5b0d3c2687f15f Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 15 Jul 2016 22:35:03 +1200 Subject: Zap unused test scripts --- test/mitmproxy/builtins/test_script.py | 14 +++++++++----- test/mitmproxy/data/scripts/a.py | 20 -------------------- test/mitmproxy/data/scripts/a_helper.py | 4 ---- test/mitmproxy/data/scripts/duplicate_flow.py | 6 ------ test/mitmproxy/data/scripts/loaderr.py | 3 --- test/mitmproxy/data/scripts/reqerr.py | 2 -- test/mitmproxy/data/scripts/starterr.py | 3 --- test/mitmproxy/data/scripts/stream_modify.py | 7 ------- test/mitmproxy/data/scripts/syntaxerr.py | 3 --- test/mitmproxy/data/scripts/tcp_stream_modify.py | 4 ---- test/mitmproxy/data/scripts/unloaderr.py | 2 -- 11 files changed, 9 insertions(+), 59 deletions(-) delete mode 100644 test/mitmproxy/data/scripts/a.py delete mode 100644 test/mitmproxy/data/scripts/a_helper.py delete mode 100644 test/mitmproxy/data/scripts/duplicate_flow.py delete mode 100644 test/mitmproxy/data/scripts/loaderr.py delete mode 100644 test/mitmproxy/data/scripts/reqerr.py delete mode 100644 test/mitmproxy/data/scripts/starterr.py delete mode 100644 test/mitmproxy/data/scripts/stream_modify.py delete mode 100644 test/mitmproxy/data/scripts/syntaxerr.py delete mode 100644 test/mitmproxy/data/scripts/tcp_stream_modify.py delete mode 100644 test/mitmproxy/data/scripts/unloaderr.py diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index da60b64c..c9616249 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -27,15 +27,19 @@ class TestParseCommand: def test_parse_args(self): with tutils.chdir(tutils.test_data.dirname): - assert script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", []) - assert script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"]) - assert script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"]) + assert script.parse_command("data/addonscripts/recorder.py") == ("data/addonscripts/recorder.py", []) + assert script.parse_command("data/addonscripts/recorder.py foo bar") == ("data/addonscripts/recorder.py", ["foo", "bar"]) + assert script.parse_command("data/addonscripts/recorder.py 'foo bar'") == ("data/addonscripts/recorder.py", ["foo bar"]) @tutils.skip_not_windows def test_parse_windows(self): with tutils.chdir(tutils.test_data.dirname): - assert script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", []) - assert script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar']) + assert script.parse_command( + "data\\addonscripts\\recorder.py" + ) == ("data\\addonscripts\\recorder.py", []) + assert script.parse_command( + "data\\addonscripts\\recorder.py 'foo \\ bar'" + ) == ("data\\addonscripts\\recorder.py", ['foo \\ bar']) def test_load_script(): diff --git a/test/mitmproxy/data/scripts/a.py b/test/mitmproxy/data/scripts/a.py deleted file mode 100644 index ab0dbf96..00000000 --- a/test/mitmproxy/data/scripts/a.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys - -from a_helper import parser - -var = 0 - - -def start(): - global var - var = parser.parse_args(sys.argv[1:]).var - - -def here(): - global var - var += 1 - return var - - -def errargs(): - pass diff --git a/test/mitmproxy/data/scripts/a_helper.py b/test/mitmproxy/data/scripts/a_helper.py deleted file mode 100644 index e1f1c649..00000000 --- a/test/mitmproxy/data/scripts/a_helper.py +++ /dev/null @@ -1,4 +0,0 @@ -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('--var', type=int) diff --git a/test/mitmproxy/data/scripts/duplicate_flow.py b/test/mitmproxy/data/scripts/duplicate_flow.py deleted file mode 100644 index 565b1845..00000000 --- a/test/mitmproxy/data/scripts/duplicate_flow.py +++ /dev/null @@ -1,6 +0,0 @@ -import mitmproxy - - -def request(f): - f = mitmproxy.ctx.master.duplicate_flow(f) - mitmproxy.ctx.master.replay_request(f, block=True, run_scripthooks=False) diff --git a/test/mitmproxy/data/scripts/loaderr.py b/test/mitmproxy/data/scripts/loaderr.py deleted file mode 100644 index 8dc4d56d..00000000 --- a/test/mitmproxy/data/scripts/loaderr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a = x diff --git a/test/mitmproxy/data/scripts/reqerr.py b/test/mitmproxy/data/scripts/reqerr.py deleted file mode 100644 index 7b419361..00000000 --- a/test/mitmproxy/data/scripts/reqerr.py +++ /dev/null @@ -1,2 +0,0 @@ -def request(r): - raise ValueError() diff --git a/test/mitmproxy/data/scripts/starterr.py b/test/mitmproxy/data/scripts/starterr.py deleted file mode 100644 index 28ba2ff1..00000000 --- a/test/mitmproxy/data/scripts/starterr.py +++ /dev/null @@ -1,3 +0,0 @@ - -def start(): - raise ValueError() diff --git a/test/mitmproxy/data/scripts/stream_modify.py b/test/mitmproxy/data/scripts/stream_modify.py deleted file mode 100644 index 4fbf45c2..00000000 --- a/test/mitmproxy/data/scripts/stream_modify.py +++ /dev/null @@ -1,7 +0,0 @@ -def modify(chunks): - for chunk in chunks: - yield chunk.replace(b"foo", b"bar") - - -def responseheaders(flow): - flow.response.stream = modify diff --git a/test/mitmproxy/data/scripts/syntaxerr.py b/test/mitmproxy/data/scripts/syntaxerr.py deleted file mode 100644 index 219d6b84..00000000 --- a/test/mitmproxy/data/scripts/syntaxerr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a + diff --git a/test/mitmproxy/data/scripts/tcp_stream_modify.py b/test/mitmproxy/data/scripts/tcp_stream_modify.py deleted file mode 100644 index 2281e6e6..00000000 --- a/test/mitmproxy/data/scripts/tcp_stream_modify.py +++ /dev/null @@ -1,4 +0,0 @@ -def tcp_message(flow): - message = flow.messages[-1] - if not message.from_client: - message.content = message.content.replace(b"foo", b"bar") diff --git a/test/mitmproxy/data/scripts/unloaderr.py b/test/mitmproxy/data/scripts/unloaderr.py deleted file mode 100644 index 6a48ab43..00000000 --- a/test/mitmproxy/data/scripts/unloaderr.py +++ /dev/null @@ -1,2 +0,0 @@ -def done(): - raise RuntimeError() -- cgit v1.2.3 From 532ae7dc4acfd6082eca92d0daf7608f94c46232 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 09:22:56 +1200 Subject: Add addons after initialisation in mitmproxy console The main effect of this is that we now properly capture errors on startup that were ignored before. --- mitmproxy/console/master.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 64bd9f0a..1ca15abe 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -204,8 +204,6 @@ class ConsoleMaster(flow.FlowMaster): def __init__(self, server, options): flow.FlowMaster.__init__(self, options, server, ConsoleState()) - self.addons.add(*builtins.default_addons()) - self.stream_path = None # This line is just for type hinting self.options = self.options # type: Options @@ -252,10 +250,12 @@ class ConsoleMaster(flow.FlowMaster): if options.app: self.start_app(self.options.app_host, self.options.app_port) + signals.call_in.connect(self.sig_call_in) signals.pop_view_state.connect(self.sig_pop_view_state) signals.push_view_state.connect(self.sig_push_view_state) signals.sig_add_event.connect(self.sig_add_event) + self.addons.add(*builtins.default_addons()) def __setattr__(self, name, value): self.__dict__[name] = value @@ -272,7 +272,7 @@ class ConsoleMaster(flow.FlowMaster): return super(ConsoleMaster, self).load_script(command, use_reloader) def sig_add_event(self, sender, e, level): - needed = dict(error=0, info=1, debug=2).get(level, 1) + needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) if self.options.verbosity < needed: return -- cgit v1.2.3 From 5a60f32c5510610935af1549022b3f7948002721 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 09:23:52 +1200 Subject: Fix log level handling We have 4: error, warn, info and debug. --- mitmproxy/cmdline.py | 2 +- mitmproxy/console/palettes.py | 6 +++++- mitmproxy/controller.py | 5 +++++ mitmproxy/dump.py | 2 +- mitmproxy/flow/master.py | 5 ----- mitmproxy/flow/options.py | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index 551fffa0..a29f58c6 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -284,7 +284,7 @@ def basic_options(parser): ) parser.add_argument( "-v", "--verbose", - action="store_const", dest="verbose", default=1, const=2, + action="store_const", dest="verbose", default=2, const=3, help="Increase event log verbosity." ) outfile = parser.add_mutually_exclusive_group() diff --git a/mitmproxy/console/palettes.py b/mitmproxy/console/palettes.py index 36cc3ac0..2e12338f 100644 --- a/mitmproxy/console/palettes.py +++ b/mitmproxy/console/palettes.py @@ -24,7 +24,7 @@ class Palette: # List and Connections 'method', 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', - 'error', + 'error', "warn", 'header', 'highlight', 'intercept', 'replay', 'mark', # Hex view @@ -100,6 +100,7 @@ class LowDark(Palette): code_500 = ('light red', 'default'), code_other = ('dark red', 'default'), + warn = ('brown', 'default'), error = ('light red', 'default'), header = ('dark cyan', 'default'), @@ -166,6 +167,7 @@ class LowLight(Palette): code_other = ('light red', 'default'), error = ('light red', 'default'), + warn = ('brown', 'default'), header = ('dark blue', 'default'), highlight = ('black,bold', 'default'), @@ -250,6 +252,7 @@ class SolarizedLight(LowLight): code_other = (sol_magenta, 'default'), error = (sol_red, 'default'), + warn = (sol_orange, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), @@ -299,6 +302,7 @@ class SolarizedDark(LowDark): code_other = (sol_magenta, 'default'), error = (sol_red, 'default'), + warn = (sol_orange, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 503cdcd3..129691bb 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -89,6 +89,11 @@ class Master(object): mitmproxy_ctx.master = None mitmproxy_ctx.log = None + def add_event(self, e, level="info"): + """ + level: debug, info, warn, error + """ + def add_server(self, server): # We give a Channel to the server which can be used to communicate with the master channel = Channel(self.event_queue, self.should_exit) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 999a709a..6a9bb951 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -114,7 +114,7 @@ class DumpMaster(flow.FlowMaster): raise DumpError(str(e)) def add_event(self, e, level="info"): - needed = dict(error=0, info=1, debug=2).get(level, 1) + needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) if self.options.verbosity >= needed: self.echo( e, diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index aa09e109..fe51c604 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -52,11 +52,6 @@ class FlowMaster(controller.Master): port ) - def add_event(self, e, level="info"): - """ - level: debug, info, error - """ - def get_ignore_filter(self): return self.server.config.check_ignore.patterns diff --git a/mitmproxy/flow/options.py b/mitmproxy/flow/options.py index eccba5b1..6c2e3933 100644 --- a/mitmproxy/flow/options.py +++ b/mitmproxy/flow/options.py @@ -30,7 +30,7 @@ class Options(options.Options): stickycookie=None, # type: Optional[str] stickyauth=None, # type: Optional[str] stream_large_bodies=None, # type: Optional[str] - verbosity=1, # type: int + verbosity=2, # type: int outfile=None, # type: Tuple[str, str] replay_ignore_content=False, # type: bool replay_ignore_params=(), # type: Sequence[str] -- cgit v1.2.3 From 5f23d4f5ca49a3f6cfbcfd9f4e444150a25844b0 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 10:00:34 +1200 Subject: add_event -> add_log throughout project "Event" is terribly over-loaded in the project, and "log" is straight-forward and self-explanatory. --- mitmproxy/cmdline.py | 4 ++-- mitmproxy/console/flowlist.py | 8 ++++---- mitmproxy/console/flowview.py | 2 +- mitmproxy/console/master.py | 39 +++++++++++++++++++-------------------- mitmproxy/console/signals.py | 6 +++--- mitmproxy/controller.py | 4 ++-- mitmproxy/dump.py | 6 +++--- mitmproxy/flow/master.py | 6 +++--- mitmproxy/web/master.py | 10 +++++----- test/mitmproxy/mastertest.py | 2 +- test/mitmproxy/test_dump.py | 3 ++- test/mitmproxy/test_examples.py | 2 +- test/mitmproxy/tservers.py | 2 +- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index a29f58c6..507ddfc7 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -285,7 +285,7 @@ def basic_options(parser): parser.add_argument( "-v", "--verbose", action="store_const", dest="verbose", default=2, const=3, - help="Increase event log verbosity." + help="Increase log verbosity." ) outfile = parser.add_mutually_exclusive_group() outfile.add_argument( @@ -384,7 +384,7 @@ def proxy_options(parser): help=""" Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore, but SSL connections are intercepted. The - communication contents are printed to the event log in verbose mode. + communication contents are printed to the log in verbose mode. """ ) group.add_argument( diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index 2f167a4d..ebbe8d21 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -44,11 +44,11 @@ footer = [ ] -class EventListBox(urwid.ListBox): +class LogBufferBox(urwid.ListBox): def __init__(self, master): self.master = master - urwid.ListBox.__init__(self, master.eventlist) + urwid.ListBox.__init__(self, master.logbuffer) def keypress(self, size, key): key = common.shortcuts(key) @@ -56,7 +56,7 @@ class EventListBox(urwid.ListBox): self.master.clear_events() key = None elif key == "G": - self.set_focus(len(self.master.eventlist) - 1) + self.set_focus(len(self.master.logbuffer) - 1) elif key == "g": self.set_focus(0) return urwid.ListBox.keypress(self, size, key) @@ -76,7 +76,7 @@ class BodyPile(urwid.Pile): [ FlowListBox(master), urwid.Frame( - EventListBox(master), + LogBufferBox(master), header = self.inactive_header ) ] diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index d809cf34..f4db5129 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -208,7 +208,7 @@ class FlowView(tabs.Tabs): ) except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() - signals.add_event(s, "error") + signals.add_log(s, "error") description, lines = contentviews.get_content_view( contentviews.get("Raw"), message.content, headers=message.headers ) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 1ca15abe..9633e494 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -236,8 +236,7 @@ class ConsoleMaster(flow.FlowMaster): self.palette = options.palette self.palette_transparent = options.palette_transparent - self.eventlog = options.eventlog - self.eventlist = urwid.SimpleListWalker([]) + self.logbuffer = urwid.SimpleListWalker([]) self.follow = options.follow if options.client_replay: @@ -254,7 +253,7 @@ class ConsoleMaster(flow.FlowMaster): signals.call_in.connect(self.sig_call_in) signals.pop_view_state.connect(self.sig_pop_view_state) signals.push_view_state.connect(self.sig_push_view_state) - signals.sig_add_event.connect(self.sig_add_event) + signals.sig_add_log.connect(self.sig_add_log) self.addons.add(*builtins.default_addons()) def __setattr__(self, name, value): @@ -271,7 +270,7 @@ class ConsoleMaster(flow.FlowMaster): # We default to using the reloader in the console ui. return super(ConsoleMaster, self).load_script(command, use_reloader) - def sig_add_event(self, sender, e, level): + def sig_add_log(self, sender, e, level): needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) if self.options.verbosity < needed: return @@ -280,13 +279,13 @@ class ConsoleMaster(flow.FlowMaster): e = urwid.Text(("error", str(e))) else: e = urwid.Text(str(e)) - self.eventlist.append(e) - if len(self.eventlist) > EVENTLOG_SIZE: - self.eventlist.pop(0) - self.eventlist.set_focus(len(self.eventlist) - 1) + self.logbuffer.append(e) + if len(self.logbuffer) > EVENTLOG_SIZE: + self.logbuffer.pop(0) + self.logbuffer.set_focus(len(self.logbuffer) - 1) - def add_event(self, e, level): - signals.add_event(e, level) + def add_log(self, e, level): + signals.add_log(e, level) def sig_call_in(self, sender, seconds, callback, args=()): def cb(*_): @@ -317,16 +316,16 @@ class ConsoleMaster(flow.FlowMaster): status, val = s.run(method, f) if val: if status: - signals.add_event("Method %s return: %s" % (method, val), "debug") + signals.add_log("Method %s return: %s" % (method, val), "debug") else: - signals.add_event( + signals.add_log( "Method %s error: %s" % (method, val[1]), "error") def run_script_once(self, command, f): if not command: return - signals.add_event("Running script on flow: %s" % command, "debug") + signals.add_log("Running script on flow: %s" % command, "debug") try: s = script.Script(command) @@ -335,7 +334,7 @@ class ConsoleMaster(flow.FlowMaster): signals.status_message.send( message='Error loading "{}".'.format(command) ) - signals.add_event('Error loading "{}":\n{}'.format(command, e), "error") + signals.add_log('Error loading "{}":\n{}'.format(command, e), "error") return if f.request: @@ -348,7 +347,7 @@ class ConsoleMaster(flow.FlowMaster): signals.flow_change.send(self, flow = f) def toggle_eventlog(self): - self.eventlog = not self.eventlog + self.options.eventlog = not self.options.eventlog signals.pop_view_state.send(self) self.view_flowlist() @@ -475,7 +474,7 @@ class ConsoleMaster(flow.FlowMaster): if self.options.rfile: ret = self.load_flows_path(self.options.rfile) if ret and self.state.flow_count(): - signals.add_event( + signals.add_log( "File truncated or corrupted. " "Loaded as many flows as possible.", "error" @@ -578,7 +577,7 @@ class ConsoleMaster(flow.FlowMaster): if self.state.follow_focus: self.state.set_focus(self.state.flow_count()) - if self.eventlog: + if self.options.eventlog: body = flowlist.BodyPile(self) else: body = flowlist.FlowListBox(self) @@ -723,7 +722,7 @@ class ConsoleMaster(flow.FlowMaster): signals.flow_change.send(self, flow = f) def clear_events(self): - self.eventlist[:] = [] + self.logbuffer[:] = [] # Handlers @controller.handler @@ -752,12 +751,12 @@ class ConsoleMaster(flow.FlowMaster): super(ConsoleMaster, self).tcp_message(f) message = f.messages[-1] direction = "->" if message.from_client else "<-" - self.add_event("{client} {direction} tcp {direction} {server}".format( + self.add_log("{client} {direction} tcp {direction} {server}".format( client=repr(f.client_conn.address), server=repr(f.server_conn.address), direction=direction, ), "info") - self.add_event(strutils.bytes_to_escaped_str(message.content), "debug") + self.add_log(strutils.bytes_to_escaped_str(message.content), "debug") @controller.handler def script_change(self, script): diff --git a/mitmproxy/console/signals.py b/mitmproxy/console/signals.py index b57ebf0c..97507834 100644 --- a/mitmproxy/console/signals.py +++ b/mitmproxy/console/signals.py @@ -3,11 +3,11 @@ from __future__ import absolute_import, print_function, division import blinker # Show a status message in the action bar -sig_add_event = blinker.Signal() +sig_add_log = blinker.Signal() -def add_event(e, level): - sig_add_event.send( +def add_log(e, level): + sig_add_log.send( None, e=e, level=level diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 129691bb..54d75e6b 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -47,7 +47,7 @@ class Log(object): self.master = master def __call__(self, text, level="info"): - self.master.add_event(text, level) + self.master.add_log(text, level) def debug(self, txt): self(txt, "debug") @@ -89,7 +89,7 @@ class Master(object): mitmproxy_ctx.master = None mitmproxy_ctx.log = None - def add_event(self, e, level="info"): + def add_log(self, e, level="info"): """ level: debug, info, warn, error """ diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 6a9bb951..70c9bb53 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -97,7 +97,7 @@ class DumpMaster(flow.FlowMaster): try: self.load_flows_file(options.rfile) except exceptions.FlowReadException as v: - self.add_event("Flow file corrupted.", "error") + self.add_log("Flow file corrupted.", "error") raise DumpError(v) if self.options.app: @@ -113,7 +113,7 @@ class DumpMaster(flow.FlowMaster): except exceptions.FlowReadException as e: raise DumpError(str(e)) - def add_event(self, e, level="info"): + def add_log(self, e, level="info"): needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) if self.options.verbosity >= needed: self.echo( @@ -157,7 +157,7 @@ class DumpMaster(flow.FlowMaster): ) except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() - self.add_event(s, "debug") + self.add_log(s, "debug") type, lines = contentviews.get_content_view( contentviews.get("Raw"), message.content, diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index fe51c604..b52e8cb6 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -282,7 +282,7 @@ class FlowMaster(controller.Master): @controller.handler def log(self, l): - self.add_event(l.msg, l.level) + self.add_log(l.msg, l.level) @controller.handler def clientconnect(self, root_layer): @@ -322,7 +322,7 @@ class FlowMaster(controller.Master): **{"mitmproxy.master": self} ) if err: - self.add_event("Error in wsgi app. %s" % err, "error") + self.add_log("Error in wsgi app. %s" % err, "error") f.reply.kill() return if f not in self.state.flows: # don't add again on replay @@ -377,7 +377,7 @@ class FlowMaster(controller.Master): @controller.handler def tcp_error(self, flow): - self.add_event("Error in TCP connection to {}: {}".format( + self.add_log("Error in TCP connection to {}: {}".format( repr(flow.server_conn.address), flow.error ), "info") diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index adfc23c4..83f18539 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -67,7 +67,7 @@ class WebState(flow.State): self._last_event_id = 0 self.events = collections.deque(maxlen=1000) - def add_event(self, e, level): + def add_log(self, e, level): self._last_event_id += 1 entry = { "id": self._last_event_id, @@ -145,7 +145,7 @@ class WebMaster(flow.FlowMaster): try: self.load_flows_file(options.rfile) except exceptions.FlowReadException as v: - self.add_event( + self.add_log( "Could not read flow file: %s" % v, "error" ) @@ -200,6 +200,6 @@ class WebMaster(flow.FlowMaster): super(WebMaster, self).error(f) return self._process_flow(f) - def add_event(self, e, level="info"): - super(WebMaster, self).add_event(e, level) - return self.state.add_event(e, level) + def add_log(self, e, level="info"): + super(WebMaster, self).add_log(e, level) + return self.state.add_log(e, level) diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 240f6a73..d1fe8cb4 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -47,5 +47,5 @@ class RecordingMaster(master.FlowMaster): master.FlowMaster.__init__(self, *args, **kwargs) self.event_log = [] - def add_event(self, e, level): + def add_log(self, e, level): self.event_log.append((level, e)) diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 201386e3..c94630a9 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -235,7 +235,8 @@ class TestDumpMaster(mastertest.MasterTest): ret = self.dummy_cycle( self.mkmaster( None, - scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1 + scripts=[tutils.test_data.path("data/scripts/all.py")], + verbosity=2 ), 1, b"", ) diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index f8646336..0ec85f52 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -21,7 +21,7 @@ class ScriptError(Exception): class RaiseMaster(master.FlowMaster): - def add_event(self, e, level): + def add_log(self, e, level): if level in ("warn", "error"): raise ScriptError(e) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 69a50b9d..9a66984b 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -42,7 +42,7 @@ class TestMaster(flow.FlowMaster): def clear_log(self): self.tlog = [] - def add_event(self, message, level=None): + def add_log(self, message, level=None): self.tlog.append(message) -- cgit v1.2.3 From 91e9016ce5af67c4f9f48ee28a9fb14736003f53 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 10:21:00 +1200 Subject: Trigger "start" when scripts are reloaded. --- mitmproxy/builtins/script.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index 14f7dd5f..ab068e47 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -131,6 +131,7 @@ class Script: self.should_reload.clear() ctx.log.info("Reloading script: %s" % self.name) self.ns = load_script(self.path, self.args) + self.start() self.configure(self.last_options) else: self.run("tick") -- cgit v1.2.3 From 49b65d2ea4418010e2fda954d9f02ecfa04e46c5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 10:33:50 +1200 Subject: Add a helper to translate a log level to a tier This is in utils for now - we'll find a better home down the track. --- mitmproxy/console/master.py | 4 ++-- mitmproxy/dump.py | 4 ++-- mitmproxy/utils.py | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 9633e494..f59729ea 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -22,6 +22,7 @@ from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import script +from mitmproxy import utils from mitmproxy.console import flowlist from mitmproxy.console import flowview from mitmproxy.console import grideditor @@ -271,8 +272,7 @@ class ConsoleMaster(flow.FlowMaster): return super(ConsoleMaster, self).load_script(command, use_reloader) def sig_add_log(self, sender, e, level): - needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) - if self.options.verbosity < needed: + if self.options.verbosity < utils.log_tier(level): return if level == "error": diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 70c9bb53..b95d2627 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -15,6 +15,7 @@ from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import flow from mitmproxy import builtins +from mitmproxy import utils from netlib import human from netlib import tcp from netlib import strutils @@ -114,8 +115,7 @@ class DumpMaster(flow.FlowMaster): raise DumpError(str(e)) def add_log(self, e, level="info"): - needed = dict(error=0, warn=1, info=2, debug=3).get(level, 2) - if self.options.verbosity >= needed: + if self.options.verbosity >= utils.log_tier(level): self.echo( e, fg="red" if level == "error" else None, diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py index 15785c72..1c75dd83 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils.py @@ -36,3 +36,7 @@ class LRUCache: d = self.cacheList.pop() self.cache.pop(d) return ret + + +def log_tier(level): + return dict(error=0, warn=1, info=2, debug=3).get(level) -- cgit v1.2.3 From 33d03d916a4866ea59bb530547f028b72d34792a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 10:39:49 +1200 Subject: Flash a status bar message if an error log event occurs --- mitmproxy/console/master.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index f59729ea..7192c281 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -276,6 +276,9 @@ class ConsoleMaster(flow.FlowMaster): return if level == "error": + signals.status_message.send( + message = "Error: %s" % str(e) + ) e = urwid.Text(("error", str(e))) else: e = urwid.Text(str(e)) -- cgit v1.2.3 From 50f1495f29adee2d3eda011693518ca5ba938037 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 10:56:33 +1200 Subject: Make mitmdump return an error exit status if errors occurred during run --- mitmproxy/dump.py | 3 +++ mitmproxy/main.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index b95d2627..18c24d61 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -45,6 +45,7 @@ class DumpMaster(flow.FlowMaster): def __init__(self, server, options): flow.FlowMaster.__init__(self, options, server, flow.State()) + self.has_errored = False self.addons.add(*builtins.default_addons()) # This line is just for type hinting self.options = self.options # type: Options @@ -115,6 +116,8 @@ class DumpMaster(flow.FlowMaster): raise DumpError(str(e)) def add_log(self, e, level="info"): + if level == "error": + self.has_errored = True if self.options.verbosity >= utils.log_tier(level): self.echo( e, diff --git a/mitmproxy/main.py b/mitmproxy/main.py index 5ced709b..316db91a 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -118,6 +118,9 @@ def mitmdump(args=None): # pragma: no cover sys.exit(1) except (KeyboardInterrupt, _thread.error): pass + if master.has_errored: + print("mitmdump: errors occurred during run", file=sys.stderr) + sys.exit(1) def mitmweb(args=None): # pragma: no cover -- cgit v1.2.3 From 17305643bc482c0b185eec5c64d506790cd26587 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Jul 2016 11:48:16 +1200 Subject: console: don't let messages overwrite prompts This renders mitmproxy unresponsive and is bad UX as well. --- mitmproxy/console/statusbar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index 040ea26d..47cc99f8 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -28,9 +28,10 @@ class ActionBar(urwid.WidgetWrap): self.pathprompt = False def sig_message(self, sender, message, expire=None): + if self.prompting: + return w = urwid.Text(message) self._w = w - self.prompting = False if expire: def cb(*args): if w == self._w: -- cgit v1.2.3