diff options
| -rw-r--r-- | .travis.yml | 33 | ||||
| -rw-r--r-- | examples/nonblocking.py | 4 | ||||
| -rw-r--r-- | examples/redirect_requests.py | 2 | ||||
| -rw-r--r-- | examples/tls_passthrough.py | 2 | ||||
| -rw-r--r-- | mitmproxy/console/statusbar.py | 4 | ||||
| -rw-r--r-- | mitmproxy/contentviews.py | 11 | ||||
| -rw-r--r-- | mitmproxy/controller.py | 59 | ||||
| -rw-r--r-- | mitmproxy/flow/master.py | 56 | ||||
| -rw-r--r-- | mitmproxy/models/flow.py | 5 | ||||
| -rw-r--r-- | mitmproxy/script/concurrent.py | 65 | ||||
| -rw-r--r-- | pathod/language/http2.py | 16 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/mastertest.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/script/test_concurrent.py | 26 | ||||
| -rw-r--r-- | test/mitmproxy/test_contentview.py | 7 | ||||
| -rw-r--r-- | test/mitmproxy/test_controller.py | 10 | ||||
| -rw-r--r-- | test/mitmproxy/test_flow.py | 8 | ||||
| -rw-r--r-- | test/mitmproxy/test_server.py | 11 | ||||
| -rw-r--r-- | test/pathod/test_language_http2.py | 66 | ||||
| -rw-r--r-- | tox.ini | 28 |
20 files changed, 208 insertions, 209 deletions
diff --git a/.travis.yml b/.travis.yml index dad81c37..b0a4ea10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,20 +14,21 @@ matrix: fast_finish: true include: - python: 2.7 + env: TOXENV=py27 - python: 2.7 - env: NO_ALPN=1 + env: TOXENV=py27 NO_ALPN=1 + - python: 3.5 + env: TOXENV=py35 + - python: 3.5 + env: TOXENV=py35 NO_ALPN=1 - language: generic + env: TOXENV=py27 os: osx osx_image: xcode7.1 git: depth: 9999999 - - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py" - - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py" NO_ALPN=1 - python: 2.7 - env: DOCS=1 - script: 'cd docs && SPHINXOPTS="-W" make -e html' + env: TOXENV=docs allow_failures: - python: pypy @@ -39,23 +40,20 @@ install: brew outdated openssl || brew upgrade openssl brew install python fi - - pip install -U virtualenv codecov - - ./dev.sh - - source venv/bin/activate before_script: - - "openssl version -a" - - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\"" - - "flake8 --jobs 4 --count mitmproxy netlib pathod examples test" + - "pip install tox" + - "tox -e lint" -script: - - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod test/$SCOPE" +script: tox after_success: - - codecov - | if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then + pip install -U virtualenv + ./dev.sh + source venv/bin/activate pip install -e ./release python ./release/rtool.py bdist python ./release/rtool.py upload-snapshot --bdist --wheel @@ -71,7 +69,6 @@ notifications: cache: directories: - - $HOME/build/mitmproxy/mitmproxy/venv + - $HOME/build/mitmproxy/mitmproxy/.tox - $HOME/.cache/pip - $HOME/.pyenv - - $HOME/Library/Caches/pip diff --git a/examples/nonblocking.py b/examples/nonblocking.py index 41674b2a..4609f389 100644 --- a/examples/nonblocking.py +++ b/examples/nonblocking.py @@ -4,6 +4,6 @@ from mitmproxy.script import concurrent @concurrent # Remove this and see what happens def request(context, flow): - print("handle request: %s%s" % (flow.request.host, flow.request.path)) + context.log("handle request: %s%s" % (flow.request.host, flow.request.path)) time.sleep(5) - print("start request: %s%s" % (flow.request.host, flow.request.path)) + context.log("start request: %s%s" % (flow.request.host, flow.request.path)) diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index 3ff8f9e4..d7db3f1c 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -16,7 +16,7 @@ def request(context, flow): "HTTP/1.1", 200, "OK", Headers(Content_Type="text/html"), "helloworld") - flow.reply(resp) + flow.reply.send(resp) # Method 2: Redirect the request to a different server if flow.request.pretty_host.endswith("example.org"): diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py index 23afe3ff..0c6d450d 100644 --- a/examples/tls_passthrough.py +++ b/examples/tls_passthrough.py @@ -134,5 +134,5 @@ def next_layer(context, next_layer): # We don't intercept - reply with a pass-through layer and add a "skipped" entry. context.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") next_layer_replacement = RawTCPLayer(next_layer.ctx, logging=False) - next_layer.reply(next_layer_replacement) + next_layer.reply.send(next_layer_replacement) context.tls_strategy.record_skipped(server_address) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index af8089b6..e576b565 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -4,7 +4,7 @@ import os.path import urwid -import netlib.utils +import netlib.http.url from mitmproxy.console import common from mitmproxy.console import pathedit from mitmproxy.console import signals @@ -212,7 +212,7 @@ class StatusBar(urwid.WidgetWrap): if self.master.server.config.mode in ["reverse", "upstream"]: dst = self.master.server.config.upstream_server - r.append("[dest:%s]" % netlib.utils.unparse( + r.append("[dest:%s]" % netlib.http.url.unparse( dst.scheme, dst.address.host, dst.address.port diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 28c57f06..006967d7 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -62,11 +62,18 @@ KEY_MAX = 30 def pretty_json(s): + # type: (bytes) -> bytes try: p = json.loads(s) except ValueError: return None - return json.dumps(p, sort_keys=True, indent=4) + pretty = json.dumps(p, sort_keys=True, indent=4, ensure_ascii=False) + if isinstance(pretty, six.text_type): + # json.dumps _may_ decide to return unicode, if the JSON object is not ascii. + # From limited testing this is always valid utf8 (otherwise json.loads will fail earlier), + # so we can just re-encode it here. + return pretty.encode("utf8", "strict") + return pretty def format_dict(d): @@ -153,7 +160,7 @@ class ViewRaw(View): content_types = [] def __call__(self, data, **metadata): - return "Raw", format_text(data) + return "Raw", format_text(strutils.bytes_to_escaped_str(data)) class ViewHex(View): diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 1498c3ad..084702a6 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -145,27 +145,6 @@ class Channel(object): self.q.put((mtype, m)) -class DummyReply(object): - """ - A reply object that does nothing. Useful when we need an object to seem - like it has a channel, and during testing. - """ - def __init__(self): - self.acked = False - self.taken = False - self.handled = False - - def take(self): - self.taken = True - - def __call__(self, msg=False): - self.acked = True - - -# Special value to distinguish the case where no reply was sent -NO_REPLY = object() - - def handler(f): @functools.wraps(f) def wrapper(*args, **kwargs): @@ -192,7 +171,7 @@ def handler(f): ret = f(*args, **kwargs) if handling and not message.reply.acked and not message.reply.taken: - message.reply() + message.reply.ack() return ret # Mark this function as a handler wrapper wrapper.func_dict["__handler"] = True @@ -215,19 +194,45 @@ class Reply(object): # Has a handler taken responsibility for ack-ing? self.handled = False + def ack(self): + self.send(self.obj) + + def kill(self): + self.send(exceptions.Kill) + def take(self): self.taken = True - def __call__(self, msg=NO_REPLY): + def send(self, msg): if self.acked: raise exceptions.ControlException("Message already acked.") self.acked = True - if msg is NO_REPLY: - self.q.put(self.obj) - else: - self.q.put(msg) + self.q.put(msg) def __del__(self): if not self.acked: # This will be ignored by the interpreter, but emit a warning raise exceptions.ControlException("Un-acked message") + + +class DummyReply(object): + """ + A reply object that does nothing. Useful when we need an object to seem + like it has a channel, and during testing. + """ + def __init__(self): + self.acked = False + self.taken = False + self.handled = False + + def kill(self): + self.send(None) + + def ack(self): + self.send(None) + + def take(self): + self.taken = True + + def send(self, msg): + self.acked = True diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index ec0bf36d..289102a1 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -103,9 +103,10 @@ class FlowMaster(controller.Master): except script.ScriptException as e: self.add_event("Script error:\n{}".format(e), "error") - def run_script_hook(self, name, *args, **kwargs): + def run_scripts(self, name, msg): for script_obj in self.scripts: - self._run_single_script_hook(script_obj, name, *args, **kwargs) + if not msg.reply.acked: + self._run_single_script_hook(script_obj, name, msg) def get_ignore_filter(self): return self.server.config.check_ignore.patterns @@ -373,28 +374,28 @@ class FlowMaster(controller.Master): @controller.handler def clientconnect(self, root_layer): - self.run_script_hook("clientconnect", root_layer) + self.run_scripts("clientconnect", root_layer) @controller.handler def clientdisconnect(self, root_layer): - self.run_script_hook("clientdisconnect", root_layer) + self.run_scripts("clientdisconnect", root_layer) @controller.handler def serverconnect(self, server_conn): - self.run_script_hook("serverconnect", server_conn) + self.run_scripts("serverconnect", server_conn) @controller.handler def serverdisconnect(self, server_conn): - self.run_script_hook("serverdisconnect", server_conn) + self.run_scripts("serverdisconnect", server_conn) @controller.handler def next_layer(self, top_layer): - self.run_script_hook("next_layer", top_layer) + self.run_scripts("next_layer", top_layer) @controller.handler def error(self, f): self.state.update_flow(f) - self.run_script_hook("error", f) + self.run_scripts("error", f) if self.client_playback: self.client_playback.clear(f) return f @@ -411,15 +412,19 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(exceptions.Kill) + f.reply.kill() return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) self.active_flows.add(f) - self.replacehooks.run(f) - self.setheaders.run(f) - self.process_new_request(f) - self.run_script_hook("request", f) + if not f.reply.acked: + self.replacehooks.run(f) + if not f.reply.acked: + self.setheaders.run(f) + if not f.reply.acked: + self.process_new_request(f) + if not f.reply.acked: + self.run_scripts("request", f) return f @controller.handler @@ -428,20 +433,23 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.exceptions.HttpException: - f.reply(exceptions.Kill) + f.reply.kill() return - self.run_script_hook("responseheaders", f) + self.run_scripts("responseheaders", f) return f @controller.handler def response(self, f): self.active_flows.discard(f) self.state.update_flow(f) - self.replacehooks.run(f) - self.setheaders.run(f) - self.run_script_hook("response", f) - if self.client_playback: - self.client_playback.clear(f) + if not f.reply.acked: + self.replacehooks.run(f) + if not f.reply.acked: + self.setheaders.run(f) + self.run_scripts("response", f) + if not f.reply.acked: + if self.client_playback: + self.client_playback.clear(f) self.process_new_response(f) if self.stream: self.stream.add(f) @@ -487,11 +495,11 @@ class FlowMaster(controller.Master): # TODO: This would break mitmproxy currently. # self.state.add_flow(flow) self.active_flows.add(flow) - self.run_script_hook("tcp_open", flow) + self.run_scripts("tcp_open", flow) @controller.handler def tcp_message(self, flow): - self.run_script_hook("tcp_message", flow) + self.run_scripts("tcp_message", flow) message = flow.messages[-1] direction = "->" if message.from_client else "<-" self.add_event("{client} {direction} tcp {direction} {server}".format( @@ -507,14 +515,14 @@ class FlowMaster(controller.Master): repr(flow.server_conn.address), flow.error ), "info") - self.run_script_hook("tcp_error", flow) + self.run_scripts("tcp_error", flow) @controller.handler def tcp_close(self, flow): self.active_flows.discard(flow) if self.stream: self.stream.add(flow) - self.run_script_hook("tcp_close", flow) + self.run_scripts("tcp_close", flow) def shutdown(self): super(FlowMaster, self).shutdown() diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py index e2dac221..de86e451 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/models/flow.py @@ -4,7 +4,6 @@ import time import copy import uuid -from mitmproxy import exceptions from mitmproxy import stateobject from mitmproxy import version from mitmproxy.models.connections import ClientConnection @@ -155,7 +154,7 @@ class Flow(stateobject.StateObject): """ self.error = Error("Connection killed") self.intercepted = False - self.reply(exceptions.Kill) + self.reply.kill() master.error(self) def intercept(self, master): @@ -175,5 +174,5 @@ class Flow(stateobject.StateObject): if not self.intercepted: return self.intercepted = False - self.reply() + self.reply.ack() master.handle_accept_intercept(self) diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index 43d0d328..89c835f6 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -4,62 +4,25 @@ offload computations from mitmproxy's main master thread. """ from __future__ import absolute_import, print_function, division +from mitmproxy import controller import threading -class ReplyProxy(object): - - def __init__(self, reply_func, script_thread): - self.reply_func = reply_func - self.script_thread = script_thread - self.master_reply = None - - def __call__(self, *args): - if self.master_reply is None: - self.master_reply = args - self.script_thread.start() - return - self.reply_func(*args) - - def done(self): - self.reply_func(*self.master_reply) - - def __getattr__(self, k): - return getattr(self.reply_func, k) - - -def _handle_concurrent_reply(fn, o, *args, **kwargs): - # Make first call to o.reply a no op and start the script thread. - # We must not start the script thread before, as this may lead to a nasty race condition - # where the script thread replies a different response before the normal reply, which then gets swallowed. - - def run(): - fn(*args, **kwargs) - # If the script did not call .reply(), we have to do it now. - reply_proxy.done() - - script_thread = ScriptThread(target=run) - - reply_proxy = ReplyProxy(o.reply, script_thread) - o.reply = reply_proxy - - class ScriptThread(threading.Thread): name = "ScriptThread" def concurrent(fn): - if fn.__name__ in ( - "request", - "response", - "error", - "clientconnect", - "serverconnect", - "clientdisconnect", - "next_layer"): - def _concurrent(ctx, obj): - _handle_concurrent_reply(fn, obj, ctx, obj) - - return _concurrent - raise NotImplementedError( - "Concurrent decorator not supported for '%s' method." % fn.__name__) + if fn.__name__ not in controller.Events: + raise NotImplementedError( + "Concurrent decorator not supported for '%s' method." % fn.__name__ + ) + + def _concurrent(ctx, obj): + def run(): + fn(ctx, obj) + if not obj.reply.acked: + obj.reply.ack() + obj.reply.take() + ScriptThread(target=run).start() + return _concurrent diff --git a/pathod/language/http2.py b/pathod/language/http2.py index 85d9047f..ea4fcd27 100644 --- a/pathod/language/http2.py +++ b/pathod/language/http2.py @@ -60,7 +60,7 @@ class _HTTP2Message(message.Message): headers = self.toks(_HeaderMixin) if not self.raw: - if not get_header("content-length", headers): + if not get_header(b"content-length", headers): if not self.body: length = 0 else: @@ -125,7 +125,7 @@ class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): def values(self, settings): value = self.value.val if self.option_used: - value = user_agents.get_by_shortcut(value.lower())[2] + value = user_agents.get_by_shortcut(value.lower().decode())[2].encode() return ( self.key.get_generator(settings), @@ -190,7 +190,7 @@ class Response(_HTTP2Message): resp = http.Response( (2, 0), self.status_code.string(), - '', + b'', headers, body, ) @@ -262,7 +262,7 @@ class Request(_HTTP2Message): else: path = self.path.string() if self.nested_response: - path += self.nested_response.parsed.spec() + path += self.nested_response.parsed.spec().encode() headers = Headers([header.values(settings) for header in self.headers]) @@ -271,11 +271,11 @@ class Request(_HTTP2Message): body = body.string() req = http.Request( - '', + b'', self.method.string(), - '', - '', - '', + b'', + b'', + b'', path, (2, 0), headers, @@ -97,8 +97,8 @@ setup( "ipaddress>=1.0.15, <1.1", ], 'dev': [ + "tox>=2.3, <3", "mock>=2.0, <2.1", - "flake8>=2.5.4, <3", "pytest>=2.8.7, <2.10", "pytest-cov>=2.2.1, <2.3", "pytest-timeout>=1.0.0, <1.1", diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 9bb8826d..4d04f337 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -16,7 +16,9 @@ class MasterTest: master.request(f) if not f.error: f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content)) + f.reply.acked = False f = master.response(f) + f.client_conn.reply.acked = False master.clientdisconnect(f.client_conn) return f diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index c2f169ad..62541f3f 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,29 +1,25 @@ -from threading import Event - from mitmproxy.script import Script from test.mitmproxy import tutils +from mitmproxy import controller +import time -class Dummy: - def __init__(self, reply): - self.reply = reply +class Thing: + def __init__(self): + self.reply = controller.DummyReply() @tutils.skip_appveyor def test_concurrent(): with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py"), None) as s: - def reply(): - reply.acked.set() - reply.acked = Event() - - f1, f2 = Dummy(reply), Dummy(reply) + f1, f2 = Thing(), Thing() s.run("request", f1) - f1.reply() s.run("request", f2) - f2.reply() - assert f1.reply.acked == reply.acked - assert not reply.acked.is_set() - assert reply.acked.wait(10) + start = time.time() + while time.time() - start < 5: + if f1.reply.acked and f2.reply.acked: + return + raise ValueError("Script never acked") def test_concurrent_err(): diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index 48d6c307..f5ba45a6 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,5 +1,3 @@ -import json - from mitmproxy.exceptions import ContentViewException from netlib.http import Headers from netlib.odict import ODict @@ -279,6 +277,7 @@ def test_get_by_shortcut(): def test_pretty_json(): - s = json.dumps({"foo": 1}) - assert cv.pretty_json(s) + assert cv.pretty_json('{"foo": 1}') assert not cv.pretty_json("moo") + assert cv.pretty_json(b'{"foo" : "\xe4\xb8\x96\xe7\x95\x8c"}') # utf8 with chinese characters + assert not cv.pretty_json(b'{"foo" : "\xFF"}') diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 83ad428e..5a68e15b 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -66,7 +66,7 @@ class TestChannel(object): def reply(): m, obj = q.get() assert m == "test" - obj.reply(42) + obj.reply.send(42) Thread(target=reply).start() @@ -86,7 +86,7 @@ class TestDummyReply(object): def test_simple(self): reply = controller.DummyReply() assert not reply.acked - reply() + reply.ack() assert reply.acked @@ -94,16 +94,16 @@ class TestReply(object): def test_simple(self): reply = controller.Reply(42) assert not reply.acked - reply("foo") + reply.send("foo") assert reply.acked assert reply.q.get() == "foo" def test_default(self): reply = controller.Reply(42) - reply() + reply.ack() assert reply.q.get() == 42 def test_reply_none(self): reply = controller.Reply(42) - reply(None) + reply.send(None) assert reply.q.get() is None diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 1b1f03f9..af8256c4 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -807,17 +807,22 @@ class TestFlowMaster: fm.load_script(tutils.test_data.path("data/scripts/all.py")) f = tutils.tflow(resp=True) + f.client_conn.acked = False fm.clientconnect(f.client_conn) assert fm.scripts[0].ns["log"][-1] == "clientconnect" + f.server_conn.acked = False fm.serverconnect(f.server_conn) assert fm.scripts[0].ns["log"][-1] == "serverconnect" + f.reply.acked = False fm.request(f) assert fm.scripts[0].ns["log"][-1] == "request" + f.reply.acked = False fm.response(f) assert fm.scripts[0].ns["log"][-1] == "response" # load second script fm.load_script(tutils.test_data.path("data/scripts/all.py")) assert len(fm.scripts) == 2 + f.server_conn.reply.acked = False fm.clientdisconnect(f.server_conn) assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" assert fm.scripts[1].ns["log"][-1] == "clientdisconnect" @@ -828,6 +833,7 @@ class TestFlowMaster: fm.load_script(tutils.test_data.path("data/scripts/all.py")) f.error = tutils.terr() + f.reply.acked = False fm.error(f) assert fm.scripts[0].ns["log"][-1] == "error" @@ -977,10 +983,12 @@ class TestFlowMaster: f = tutils.tflow(resp=True) f.response.headers["set-cookie"] = "foo=bar" fm.request(f) + f.reply.acked = False fm.response(f) assert fm.stickycookie_state.jar assert "cookie" not in f.request.headers f = f.copy() + f.reply.acked = False fm.request(f) assert f.request.headers["cookie"] == "foo=bar" diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index b58c4f44..432340c0 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -14,7 +14,6 @@ from pathod import pathoc, pathod from mitmproxy import controller from mitmproxy.proxy.config import HostMatcher -from mitmproxy.exceptions import Kill from mitmproxy.models import Error, HTTPResponse, HTTPFlow from . import tutils, tservers @@ -744,7 +743,7 @@ class MasterFakeResponse(tservers.TestMaster): @controller.handler def request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) - f.reply(resp) + f.reply.send(resp) class TestFakeResponse(tservers.HTTPProxyTest): @@ -771,7 +770,7 @@ class MasterKillRequest(tservers.TestMaster): @controller.handler def request(self, f): - f.reply(Kill) + f.reply.kill() class TestKillRequest(tservers.HTTPProxyTest): @@ -788,7 +787,7 @@ class MasterKillResponse(tservers.TestMaster): @controller.handler def response(self, f): - f.reply(Kill) + f.reply.kill() class TestKillResponse(tservers.HTTPProxyTest): @@ -820,7 +819,7 @@ class MasterIncomplete(tservers.TestMaster): def request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.content = None - f.reply(resp) + f.reply.send(resp) class TestIncompleteResponse(tservers.HTTPProxyTest): @@ -942,7 +941,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): if not (k[0] in exclude): f.client_conn.finish() f.error = Error("terminated") - f.reply(Kill) + f.reply.kill() return _func(f) setattr(master, attr, handler) diff --git a/test/pathod/test_language_http2.py b/test/pathod/test_language_http2.py index de256626..a2bffe63 100644 --- a/test/pathod/test_language_http2.py +++ b/test/pathod/test_language_http2.py @@ -1,4 +1,4 @@ -from six.moves import cStringIO as StringIO +from six import BytesIO import netlib from netlib import tcp @@ -10,11 +10,11 @@ import tutils def parse_request(s): - return language.parse_pathoc(s, True).next() + return next(language.parse_pathoc(s, True)) def parse_response(s): - return language.parse_pathod(s, True).next() + return next(language.parse_pathod(s, True)) def default_settings(): @@ -25,7 +25,7 @@ def default_settings(): def test_make_error_response(): - d = StringIO() + d = BytesIO() s = http2.make_error_response("foo", "bar") language.serve(s, d, default_settings()) @@ -46,15 +46,15 @@ class TestRequest: def test_simple(self): r = parse_request('GET:"/foo"') - assert r.method.string() == "GET" - assert r.path.string() == "/foo" + assert r.method.string() == b"GET" + assert r.path.string() == b"/foo" r = parse_request('GET:/foo') - assert r.path.string() == "/foo" + assert r.path.string() == b"/foo" def test_multiple(self): r = list(language.parse_pathoc("GET:/ PUT:/")) - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" assert len(r) == 2 l = """ @@ -71,8 +71,8 @@ class TestRequest: """ r = list(language.parse_pathoc(l, True)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" l = """ get:"http://localhost:9999/p/200" @@ -80,11 +80,11 @@ class TestRequest: """ r = list(language.parse_pathoc(l, True)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "GET" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"GET" def test_render_simple(self): - s = StringIO() + s = BytesIO() r = parse_request("GET:'/foo'") assert language.serve( r, @@ -101,32 +101,32 @@ class TestRequest: r = parse_request('GET:/') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-length", "0") + assert r.headers[0].values(default_settings()) == (b"content-length", b"0") r = parse_request('GET:/:b"foobar"') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-length", "6") + assert r.headers[0].values(default_settings()) == (b"content-length", b"6") r = parse_request('GET:/:b"foobar":h"content-length"="42"') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-length", "42") + assert r.headers[0].values(default_settings()) == (b"content-length", b"42") r = parse_request('GET:/:r:b"foobar":h"content-length"="42"') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-length", "42") + assert r.headers[0].values(default_settings()) == (b"content-length", b"42") def test_content_type(self): r = parse_request('GET:/:r:c"foobar"') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-type", "foobar") + assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar") def test_user_agent(self): r = parse_request('GET:/:r:ua') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("user-agent", user_agents.get_by_shortcut('a')[2]) + assert r.headers[0].values(default_settings()) == (b"user-agent", user_agents.get_by_shortcut('a')[2].encode()) def test_render_with_headers(self): - s = StringIO() + s = BytesIO() r = parse_request('GET:/foo:h"foo"="bar"') assert language.serve( r, @@ -142,7 +142,7 @@ class TestRequest: assert r.values(default_settings()) def test_render_with_body(self): - s = StringIO() + s = BytesIO() r = parse_request("GET:'/foo':bfoobar") assert language.serve( r, @@ -177,29 +177,29 @@ class TestResponse: r = parse_response('200') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-length", "0") + assert r.headers[0].values(default_settings()) == (b"content-length", b"0") def test_content_type(self): r = parse_response('200:r:c"foobar"') assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("content-type", "foobar") + assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar") def test_simple(self): r = parse_response('200:r:h"foo"="bar"') - assert r.status_code.string() == "200" + assert r.status_code.string() == b"200" assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == ("foo", "bar") + assert r.headers[0].values(default_settings()) == (b"foo", b"bar") assert r.body is None r = parse_response('200:r:h"foo"="bar":bfoobar:h"bla"="fasel"') - assert r.status_code.string() == "200" + assert r.status_code.string() == b"200" assert len(r.headers) == 2 - assert r.headers[0].values(default_settings()) == ("foo", "bar") - assert r.headers[1].values(default_settings()) == ("bla", "fasel") - assert r.body.string() == "foobar" + assert r.headers[0].values(default_settings()) == (b"foo", b"bar") + assert r.headers[1].values(default_settings()) == (b"bla", b"fasel") + assert r.body.string() == b"foobar" def test_render_simple(self): - s = StringIO() + s = BytesIO() r = parse_response('200') assert language.serve( r, @@ -208,7 +208,7 @@ class TestResponse: ) def test_render_with_headers(self): - s = StringIO() + s = BytesIO() r = parse_response('200:h"foo"="bar"') assert language.serve( r, @@ -217,7 +217,7 @@ class TestResponse: ) def test_render_with_body(self): - s = StringIO() + s = BytesIO() r = parse_response('200:bfoobar') assert language.serve( r, @@ -1,15 +1,31 @@ [tox] -envlist = py27, py35, lint +envlist = py27, py35, docs, lint [testenv] -deps = -rrequirements.txt +deps = + -rrequirements.txt + codecov>=2.0.5 +passenv = CI TRAVIS_BUILD_ID TRAVIS TRAVIS_BRANCH TRAVIS_JOB_NUMBER TRAVIS_PULL_REQUEST TRAVIS_JOB_ID TRAVIS_REPO_SLUG TRAVIS_COMMIT [testenv:py27] -commands = py.test -n 8 --timeout 60 ./test +commands = + py.test --cov netlib --cov mitmproxy --cov pathod --color=yes --timeout 60 ./test + codecov -e TOXENV [testenv:py35] -commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test_language_websocket.py +# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0 +whitelist_externals = bash +commands = + bash -c 'set -o pipefail ; py.test --cov netlib --cov mitmproxy --cov pathod --color=yes --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py test/pathod/test_language_http2.py 2>&1 | grep -v Cryptography_locking_cb' + codecov -e TOXENV + +[testenv:docs] +basepython = python2.7 +whitelist_externals = make +changedir = docs +setenv = SPHINXOPTS="-W" +commands = make -e html [testenv:lint] -deps = flake8 -commands = flake8 --count mitmproxy netlib pathod examples test +deps = flake8>=2.5.4, <3 +commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test |