diff options
| -rw-r--r-- | README.rst | 45 | ||||
| -rwxr-xr-x | dev.sh | 5 | ||||
| -rw-r--r-- | mitmproxy/console/__init__.py | 33 | ||||
| -rw-r--r-- | mitmproxy/console/flowlist.py | 7 | ||||
| -rw-r--r-- | mitmproxy/console/statusbar.py | 4 | ||||
| -rw-r--r-- | mitmproxy/dump.py | 8 | ||||
| -rw-r--r-- | netlib/tcp.py | 43 | ||||
| -rw-r--r-- | pathod/language/generators.py | 2 | ||||
| -rw-r--r-- | pathod/pathoc.py | 3 | ||||
| -rw-r--r-- | pathod/pathod.py | 18 | ||||
| -rw-r--r-- | pathod/protocols/websockets.py | 2 | ||||
| -rw-r--r-- | pathod/test.py | 49 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | test/pathod/test_app.py | 6 | ||||
| -rw-r--r-- | test/pathod/test_pathod.py | 22 | ||||
| -rw-r--r-- | test/pathod/test_test.py | 4 | ||||
| -rw-r--r-- | test/pathod/tutils.py | 48 |
17 files changed, 211 insertions, 91 deletions
@@ -3,19 +3,24 @@ mitmproxy |travis| |coveralls| |latest_release| |python_versions| -This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**. +This repository contains the **mitmproxy** and **pathod** projects, as well as +their shared networking library, **netlib**. -``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface. +``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console +interface. ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP. -``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards. +``pathoc`` and ``pathod`` are perverse HTTP client and server applications +designed to let you craft almost any conceivable HTTP request, including ones +that creatively violate the standards. Documentation & Help -------------------- -Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites. +Documentation, tutorials and precompiled binaries can be found on the mitmproxy +and pathod websites. |mitmproxy_site| |pathod_site| @@ -28,12 +33,19 @@ You can join our developer chat on Slack. |slack| +Installation +------------ + +The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_. +If you want to contribute changes, keep on reading. + + Hacking ------- To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with -virtualenv_ installed (you can find installation instructions for virtualenv here_). -Then do the following: +virtualenv_ installed (you can find installation instructions for virtualenv +here_). Then do the following: .. code-block:: text @@ -42,10 +54,11 @@ Then do the following: ./dev.sh -The *dev* script will create a virtualenv environment in a directory called "venv", -and install all mandatory and optional dependencies into it. -The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable", -so any changes to the source in the repository will be reflected live in the virtualenv. +The *dev* script will create a virtualenv environment in a directory called +"venv", and install all mandatory and optional dependencies into it. The +primary mitmproxy components - mitmproxy, netlib and pathod - are installed as +"editable", so any changes to the source in the repository will be reflected +live in the virtualenv. To confirm that you're up and running, activate the virtualenv, and run the mitmproxy test suite: @@ -56,9 +69,9 @@ mitmproxy test suite: py.test Note that the main executables for the project - ``mitmdump``, ``mitmproxy``, -``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the -virtualenv, they will be on your $PATH, and you can run them like any other -command: +``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the +virtualenv. After activating the virtualenv, they will be on your $PATH, and +you can run them like any other command: .. code-block:: text @@ -85,9 +98,9 @@ suite. The project tries to maintain 100% test coverage. Documentation ------------- -The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development -environment as described above. -After installation, you can render the documentation like this: +The mitmproxy documentation is build using Sphinx_, which is installed +automatically if you set up a development environment as described above. After +installation, you can render the documentation like this: .. code-block:: text @@ -1,5 +1,6 @@ #!/bin/sh set -e +set -x PYVERSION=$1 VENV="venv$1" @@ -8,8 +9,8 @@ echo "Creating dev environment in $VENV using Python $PYVERSION" python$PYVERSION -m virtualenv "$VENV" --always-copy . "$VENV/bin/activate" -pip$PYVERSION install -q -U pip setuptools -pip$PYVERSION install -q -r requirements.txt +pip$PYVERSION install -U pip setuptools +pip$PYVERSION install -r requirements.txt echo "" echo "* Virtualenv created in $VENV and all dependencies installed." diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py index 00fb4b1b..63692ec0 100644 --- a/mitmproxy/console/__init__.py +++ b/mitmproxy/console/__init__.py @@ -44,6 +44,8 @@ class ConsoleState(flow.State): self.default_body_view = contentviews.get("Auto") self.flowsettings = weakref.WeakKeyDictionary() self.last_search = None + self.last_filter = None + self.mark_filter = False def __setattr__(self, name, value): self.__dict__[name] = value @@ -117,6 +119,37 @@ class ConsoleState(flow.State): self.set_focus(self.focus) return ret + def filter_marked(self, m): + def actual_func(x): + if x.id in m: + return True + return False + return actual_func + + def enable_marked_filter(self): + self.last_filter = self.limit_txt + marked_flows = [] + for f in self.flows: + if self.flow_marked(f): + marked_flows.append(f.id) + if len(marked_flows) > 0: + f = self.filter_marked(marked_flows) + self.view._close() + self.view = flow.FlowView(self.flows, f) + self.focus = 0 + self.set_focus(self.focus) + self.mark_filter = True + + def disable_marked_filter(self): + if self.last_filter is None: + self.view = flow.FlowView(self.flows, None) + else: + self.set_limit(self.last_filter) + self.focus = 0 + self.set_focus(self.focus) + self.last_filter = None + self.mark_filter = False + def clear(self): marked_flows = [] for f in self.flows: diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index eb1e76fb..8c20c4b6 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -22,6 +22,7 @@ def _mkhelp(): ("l", "set limit filter pattern"), ("L", "load saved flows"), ("m", "toggle flow mark"), + ("M", "toggle marked flow view"), ("n", "create a new request"), ("P", "copy flow to clipboard"), ("r", "replay request"), @@ -198,6 +199,12 @@ class ConnectionItem(urwid.WidgetWrap): else: self.state.set_flow_marked(self.flow, True) signals.flowlist_change.send(self) + elif key == "M": + if self.state.mark_filter: + self.state.disable_marked_filter() + else: + self.state.enable_marked_filter() + signals.flowlist_change.send(self) elif key == "r": r = self.master.replay_request(self.flow) if r: diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index b3e1517f..af8089b6 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -168,6 +168,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "l")) r.append(":%s]" % self.master.state.limit_txt) + if self.master.state.mark_filter: + r.append("[") + r.append(("heading_key", "Marked Flows")) + r.append("]") if self.master.stickycookie_txt: r.append("[") r.append(("heading_key", "t")) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index b1005ee7..cc6896ed 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -333,17 +333,15 @@ class DumpMaster(flow.FlowMaster): @controller.handler def request(self, f): - flow.FlowMaster.request(self, f) - self.state.delete_flow(f) + f = flow.FlowMaster.request(self, f) if f: - f.reply() + self.state.delete_flow(f) return f @controller.handler def response(self, f): - flow.FlowMaster.response(self, f) + f = flow.FlowMaster.response(self, f) if f: - f.reply() self._process_flow(f) return f diff --git a/netlib/tcp.py b/netlib/tcp.py index 914aa701..de12102e 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -6,6 +6,7 @@ import sys import threading import time import traceback +import contextlib import binascii from six.moves import range @@ -577,6 +578,12 @@ class _Connection(object): return context +@contextlib.contextmanager +def _closer(client): + yield + client.close() + + class TCPClient(_Connection): def __init__(self, address, source_address=None): @@ -708,6 +715,7 @@ class TCPClient(_Connection): self.connection = connection self.ip_address = Address(connection.getpeername()) self._makefile() + return _closer(self) def settimeout(self, n): self.connection.settimeout(n) @@ -833,6 +841,25 @@ class BaseHandler(_Connection): return b"" +class Counter: + def __init__(self): + self._count = 0 + self._lock = threading.Lock() + + @property + def count(self): + with self._lock: + return self._count + + def __enter__(self): + with self._lock: + self._count += 1 + + def __exit__(self, *args): + with self._lock: + self._count -= 1 + + class TCPServer(object): request_queue_size = 20 @@ -845,15 +872,17 @@ class TCPServer(object): self.socket.bind(self.address()) self.address = Address.wrap(self.socket.getsockname()) self.socket.listen(self.request_queue_size) + self.handler_counter = Counter() def connection_thread(self, connection, client_address): - client_address = Address(client_address) - try: - self.handle_client_connection(connection, client_address) - except: - self.handle_error(connection, client_address) - finally: - close_socket(connection) + with self.handler_counter: + client_address = Address(client_address) + try: + self.handle_client_connection(connection, client_address) + except: + self.handle_error(connection, client_address) + finally: + close_socket(connection) def serve_forever(self, poll_interval=0.1): self.__is_shut_down.clear() diff --git a/pathod/language/generators.py b/pathod/language/generators.py index 01f709e2..9fff3082 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -37,6 +37,8 @@ class TransformGenerator(object): def __getitem__(self, x): d = self.gen.__getitem__(x) + if isinstance(x, slice): + return self.transform(x.start, d) return self.transform(x, d) def __repr__(self): diff --git a/pathod/pathoc.py b/pathod/pathoc.py index 2b7d053c..5cfb4591 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -286,7 +286,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2 and not self.ssl: raise NotImplementedError("HTTP2 without SSL is not supported.") - tcp.TCPClient.connect(self) + ret = tcp.TCPClient.connect(self) if connect_to: self.http_connect(connect_to) @@ -324,6 +324,7 @@ class Pathoc(tcp.TCPClient): if self.timeout: self.settimeout(self.timeout) + return ret def stop(self): if self.ws_framereader: diff --git a/pathod/pathod.py b/pathod/pathod.py index 7795df0e..0449c0c1 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -353,6 +353,8 @@ class Pathod(tcp.TCPServer): staticdir=self.staticdir ) + self.loglock = threading.Lock() + def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. @@ -403,8 +405,7 @@ class Pathod(tcp.TCPServer): def add_log(self, d): if not self.noapi: - lock = threading.Lock() - with lock: + with self.loglock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: @@ -413,17 +414,18 @@ class Pathod(tcp.TCPServer): return d["id"] def clear_log(self): - lock = threading.Lock() - with lock: + with self.loglock: self.log = [] def log_by_id(self, identifier): - for i in self.log: - if i["id"] == identifier: - return i + with self.loglock: + for i in self.log: + if i["id"] == identifier: + return i def get_log(self): - return self.log + with self.loglock: + return self.log def main(args): # pragma: no cover diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 134d27bc..2b60e618 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -18,7 +18,7 @@ class WebsocketsProtocol: frm = websockets.Frame.from_file(self.pathod_handler.rfile) except NetlibException as e: lg("Error reading websocket frame: %s" % e) - break + return None, None ended = time.time() lg(frm.human_readable()) retlog = dict( diff --git a/pathod/test.py b/pathod/test.py index 23b7a5b6..11462729 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -1,12 +1,14 @@ from six.moves import cStringIO as StringIO import threading +import time + from six.moves import queue -import requests -import requests.packages.urllib3 from . import pathod -requests.packages.urllib3.disable_warnings() + +class TimeoutError(Exception): + pass class Daemon: @@ -39,39 +41,51 @@ class Daemon: """ return "%s/p/%s" % (self.urlbase, spec) - def info(self): - """ - Return some basic info about the remote daemon. - """ - resp = requests.get("%s/api/info" % self.urlbase, verify=False) - return resp.json() - def text_log(self): return self.logfp.getvalue() + def wait_for_silence(self, timeout=5): + start = time.time() + while 1: + if time.time() - start >= timeout: + raise TimeoutError( + "%s service threads still alive" % + self.thread.server.handler_counter.count + ) + if self.thread.server.handler_counter.count == 0: + return + + def expect_log(self, n, timeout=5): + l = [] + start = time.time() + while True: + l = self.log() + if time.time() - start >= timeout: + return None + if len(l) >= n: + break + return l + def last_log(self): """ Returns the last logged request, or None. """ - l = self.log() + l = self.expect_log(1) if not l: return None - return l[0] + return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ - resp = requests.get("%s/api/log" % self.urlbase, verify=False) - return resp.json()["log"] + return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ - self.logfp.truncate(0) - resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) - return resp.ok + return self.thread.server.clear_log() def shutdown(self): """ @@ -88,6 +102,7 @@ class _PaThread(threading.Thread): self.name = "PathodThread" self.iface, self.q, self.ssl = iface, q, ssl self.daemonargs = daemonargs + self.server = None def run(self): self.server = pathod.Pathod( @@ -11,8 +11,7 @@ addopts = --capture=no [coverage:run] branch = True -include = *mitmproxy*, *netlib*, *pathod* -omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py +omit = *contrib*, *tnetstring*, *platform*, *main.py [coverage:report] show_missing = True diff --git a/test/pathod/test_app.py b/test/pathod/test_app.py index ac89c44c..fbaa773c 100644 --- a/test/pathod/test_app.py +++ b/test/pathod/test_app.py @@ -11,11 +11,11 @@ class TestApp(tutils.DaemonTests): def test_about(self): r = self.getpath("/about") - assert r.ok + assert r.status_code == 200 def test_download(self): r = self.getpath("/download") - assert r.ok + assert r.status_code == 200 def test_docs(self): assert self.getpath("/docs/pathod").status_code == 200 @@ -27,7 +27,7 @@ class TestApp(tutils.DaemonTests): def test_log(self): assert self.getpath("/log").status_code == 200 assert self.get("200:da").status_code == 200 - id = self.d.log()[0]["id"] + id = self.d.expect_log(1)[0]["id"] assert self.getpath("/log").status_code == 200 assert self.getpath("/log/%s" % id).status_code == 200 assert self.getpath("/log/9999999").status_code == 404 diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 4d969158..ec9c169f 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -1,7 +1,6 @@ from six.moves import cStringIO as StringIO -import pytest -from pathod import pathod, version +from pathod import pathod from netlib import tcp from netlib.exceptions import HttpException, TlsException import tutils @@ -129,7 +128,6 @@ class CommonTests(tutils.DaemonTests): assert self.d.last_log() # FIXME: Other binary data elements - @pytest.mark.skip(reason="race condition") def test_sizelimit(self): r = self.get("200:b@1g") assert r.status_code == 800 @@ -140,21 +138,15 @@ class CommonTests(tutils.DaemonTests): r, _ = self.pathoc([r"get:'/p/200':i0,'\r\n'"]) assert r[0].status_code == 200 - def test_info(self): - assert tuple(self.d.info()["version"]) == version.IVERSION - - @pytest.mark.skip(reason="race condition") def test_logs(self): - assert self.d.clear_log() - assert not self.d.last_log() + self.d.clear_log() assert self.get("202:da") - assert len(self.d.log()) == 1 - assert self.d.clear_log() + assert self.d.expect_log(1) + self.d.clear_log() assert len(self.d.log()) == 0 def test_disconnect(self): - rsp = self.get("202:b@100k:d200") - assert len(rsp.content) < 200 + tutils.raises("unexpected eof", self.get, "202:b@100k:d200") def test_parserr(self): rsp = self.get("400:msg,b:") @@ -166,7 +158,7 @@ class CommonTests(tutils.DaemonTests): assert rsp.content.strip() == "testfile" def test_anchor(self): - rsp = self.getpath("anchor/foo") + rsp = self.getpath("/anchor/foo") assert rsp.status_code == 202 def test_invalid_first_line(self): @@ -223,7 +215,6 @@ class CommonTests(tutils.DaemonTests): ) assert r[1].payload == "test" - @pytest.mark.skip(reason="race condition") def test_websocket_frame_reflect_error(self): r, _ = self.pathoc( ["ws:/p/", "wf:-mask:knone:f'wf:b@10':i13,'a'"], @@ -233,7 +224,6 @@ class CommonTests(tutils.DaemonTests): # FIXME: Race Condition? assert "Parse error" in self.d.text_log() - @pytest.mark.skip(reason="race condition") def test_websocket_frame_disconnect_error(self): self.pathoc(["ws:/p/", "wf:b@10:d3"], ws_read_limit=0) assert self.d.last_log() diff --git a/test/pathod/test_test.py b/test/pathod/test_test.py index cee286a4..6399894e 100644 --- a/test/pathod/test_test.py +++ b/test/pathod/test_test.py @@ -2,6 +2,10 @@ import logging import requests from pathod import test import tutils + +import requests.packages.urllib3 + +requests.packages.urllib3.disable_warnings() logging.disable(logging.CRITICAL) diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index f7bb22e5..b9f38d86 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -3,6 +3,7 @@ import re import shutil import requests from six.moves import cStringIO as StringIO +import urllib from netlib import tcp from netlib import utils @@ -63,10 +64,11 @@ class DaemonTests(object): shutil.rmtree(cls.confdir) def teardown(self): + self.d.wait_for_silence() if not (self.noweb or self.noapi): self.d.clear_log() - def getpath(self, path, params=None): + def _getpath(self, path, params=None): scheme = "https" if self.ssl else "http" resp = requests.get( "%s://localhost:%s/%s" % ( @@ -79,9 +81,29 @@ class DaemonTests(object): ) return resp + def getpath(self, path, params=None): + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + with c.connect(): + if params: + path = path + "?" + urllib.urlencode(params) + resp = c.request("get:%s" % path) + return resp + def get(self, spec): - resp = requests.get(self.d.p(spec), verify=False) - return resp + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + with c.connect(): + resp = c.request("get:/p/%s" % urllib.quote(spec).encode("string_escape")) + return resp def pathoc( self, @@ -106,16 +128,16 @@ class DaemonTests(object): fp=logfp, use_http2=use_http2, ) - c.connect(connect_to) - ret = [] - for i in specs: - resp = c.request(i) - if resp: - ret.append(resp) - for frm in c.wait(): - ret.append(frm) - c.stop() - return ret, logfp.getvalue() + with c.connect(connect_to): + ret = [] + for i in specs: + resp = c.request(i) + if resp: + ret.append(resp) + for frm in c.wait(): + ret.append(frm) + c.stop() + return ret, logfp.getvalue() tmpdir = tutils.tmpdir |