diff options
Diffstat (limited to 'mitmproxy/tools')
| -rw-r--r-- | mitmproxy/tools/console/flowlist.py | 6 | ||||
| -rw-r--r-- | mitmproxy/tools/console/flowview.py | 6 | ||||
| -rw-r--r-- | mitmproxy/tools/console/master.py | 2 | ||||
| -rw-r--r-- | mitmproxy/tools/main.py | 1 | ||||
| -rw-r--r-- | mitmproxy/tools/web/app.py | 266 | ||||
| -rw-r--r-- | mitmproxy/tools/web/master.py | 125 |
6 files changed, 170 insertions, 236 deletions
diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 395a9d52..d7c312e5 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -303,8 +303,8 @@ class FlowListWalker(urwid.ListWalker): class FlowListBox(urwid.ListBox): - def __init__(self, master): - self.master = master + def __init__(self, master: "mitmproxy.tools.console.master.ConsoleMaster"): + self.master = master # type: "mitmproxy.tools.console.master.ConsoleMaster" super().__init__(FlowListWalker(master)) def get_method_raw(self, k): @@ -348,7 +348,7 @@ class FlowListBox(urwid.ListBox): if key == "A": for f in self.master.view: if f.intercepted: - f.resume() + f.resume(self.master) signals.flowlist_change.send(self) elif key == "z": self.master.view.clear() diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index e2b24fab..ecb070d8 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -510,8 +510,10 @@ class FlowView(tabs.Tabs): self.flow.resume(self.master) signals.flow_change.send(self, flow = self.flow) elif key == "A": - self.master.accept_all() - signals.flow_change.send(self, flow = self.flow) + for f in self.view: + if f.intercepted: + f.resume(self.master) + signals.flow_change.send(self, flow=f) elif key == "d": if self.flow.killable: self.flow.kill(self.master) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 184038ef..5d0e0ef4 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -71,7 +71,7 @@ class ConsoleMaster(master.Master): def __init__(self, options, server): super().__init__(options, server) - self.view = view.View() + self.view = view.View() # type: view.View self.stream_path = None # This line is just for type hinting self.options = self.options # type: Options diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 69dd3791..478690eb 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -135,7 +135,6 @@ def mitmweb(args=None): # pragma: no cover web_options.wdebug = args.wdebug web_options.wiface = args.wiface web_options.wport = args.wport - web_options.process_web_options(parser) server = process_options(parser, web_options, args) m = web.master.WebMaster(web_options, server) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 25a46169..f617bd08 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -1,5 +1,3 @@ - -import base64 import hashlib import json import logging @@ -7,19 +5,21 @@ import os.path import re from io import BytesIO +import mitmproxy.addons.view +import mitmproxy.flow +import tornado.escape import tornado.web import tornado.websocket -import tornado.escape from mitmproxy import contentviews +from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io +from mitmproxy import log from mitmproxy import version -import mitmproxy.addons.view -import mitmproxy.flow -def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: +def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: """ Remove flow message content and cert to save transmission space. @@ -46,8 +46,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: "path": flow.request.path, "http_version": flow.request.http_version, "headers": tuple(flow.request.headers.items(True)), - "contentLength": len(flow.request.raw_content) if flow.request.raw_content is not None else None, - "contentHash": hashlib.sha256(flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None, + "contentLength": len( + flow.request.raw_content) if flow.request.raw_content is not None else None, + "contentHash": hashlib.sha256( + flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None, "timestamp_start": flow.request.timestamp_start, "timestamp_end": flow.request.timestamp_end, "is_replay": flow.request.is_replay, @@ -58,8 +60,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: "status_code": flow.response.status_code, "reason": flow.response.reason, "headers": tuple(flow.response.headers.items(True)), - "contentLength": len(flow.response.raw_content) if flow.response.raw_content is not None else None, - "contentHash": hashlib.sha256(flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None, + "contentLength": len( + flow.response.raw_content) if flow.response.raw_content is not None else None, + "contentHash": hashlib.sha256( + flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None, "timestamp_start": flow.response.timestamp_start, "timestamp_end": flow.response.timestamp_end, "is_replay": flow.response.is_replay, @@ -69,34 +73,19 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: return f -class APIError(tornado.web.HTTPError): - pass - - -class BasicAuth: - - def set_auth_headers(self): - self.set_status(401) - self.set_header('WWW-Authenticate', 'Basic realm=MITMWeb') - self._transforms = [] - self.finish() +def logentry_to_json(e: log.LogEntry) -> dict: + return { + "id": id(e), # we just need some kind of id. + "message": e.msg, + "level": e.level + } - def prepare(self): - wauthenticator = self.application.settings['wauthenticator'] - if wauthenticator: - auth_header = self.request.headers.get('Authorization') - if auth_header is None or not auth_header.startswith('Basic '): - self.set_auth_headers() - else: - auth_decoded = base64.decodebytes(auth_header[6:]) - username, password = auth_decoded.split(':', 2) - if not wauthenticator.test(username, password): - self.set_auth_headers() - raise APIError(401, "Invalid username or password.") +class APIError(tornado.web.HTTPError): + pass -class RequestHandler(BasicAuth, tornado.web.RequestHandler): +class RequestHandler(tornado.web.RequestHandler): def write(self, chunk): # Writing arrays on the top level is ok nowadays. # http://flask.pocoo.org/docs/0.11/security/#json-security @@ -120,9 +109,23 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler): @property def json(self): - if not self.request.headers.get("Content-Type").startswith("application/json"): - return None - return json.loads(self.request.body.decode()) + if not self.request.headers.get("Content-Type", "").startswith("application/json"): + raise APIError(400, "Invalid Content-Type, expected application/json.") + try: + return json.loads(self.request.body.decode()) + except Exception as e: + raise APIError(400, "Malformed JSON: {}".format(str(e))) + + @property + def filecontents(self): + """ + Accept either a multipart/form file upload or just take the plain request body. + + """ + if self.request.files: + return next(iter(self.request.files.values()))[0].body + else: + return self.request.body @property def view(self) -> mitmproxy.addons.view.View: @@ -136,11 +139,11 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler): def flow(self) -> mitmproxy.flow.Flow: flow_id = str(self.path_kwargs["flow_id"]) # FIXME: Add a facility to addon.view to safely access the store - flow = self.view._store.get(flow_id) + flow = self.view.get_by_id(flow_id) if flow: return flow else: - raise APIError(400, "Flow not found.") + raise APIError(404, "Flow not found.") def write_error(self, status_code: int, **kwargs): if "exc_info" in kwargs and isinstance(kwargs["exc_info"][1], APIError): @@ -150,7 +153,6 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler): class IndexHandler(RequestHandler): - def get(self): token = self.xsrf_token # https://github.com/tornadoweb/tornado/issues/645 assert token @@ -158,14 +160,13 @@ class IndexHandler(RequestHandler): class FilterHelp(RequestHandler): - def get(self): self.write(dict( commands=flowfilter.help )) -class WebSocketEventBroadcaster(BasicAuth, tornado.websocket.WebSocketHandler): +class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): # raise an error if inherited class doesn't specify its own instance. connections = None # type: set @@ -182,7 +183,7 @@ class WebSocketEventBroadcaster(BasicAuth, tornado.websocket.WebSocketHandler): for conn in cls.connections: try: conn.write_message(message) - except Exception: + except Exception: # pragma: no cover logging.error("Error sending message", exc_info=True) @@ -191,9 +192,8 @@ class ClientConnection(WebSocketEventBroadcaster): class Flows(RequestHandler): - def get(self): - self.write([convert_flow_to_json_dict(f) for f in self.view]) + self.write([flow_to_json(f) for f in self.view]) class DumpFlows(RequestHandler): @@ -211,33 +211,29 @@ class DumpFlows(RequestHandler): def post(self): self.view.clear() - - content = self.request.files.values()[0][0].body - bio = BytesIO(content) - self.master.load_flows(io.FlowReader(bio).stream()) + bio = BytesIO(self.filecontents) + self.master.load_flows(io.FlowReader(bio)) bio.close() class ClearAll(RequestHandler): - def post(self): self.view.clear() + self.master.events.clear() class AcceptFlows(RequestHandler): - def post(self): - self.master.accept_all(self.master) + for f in self.view: + f.resume(self.master) class AcceptFlow(RequestHandler): - def post(self, flow_id): self.flow.resume(self.master) class FlowHandler(RequestHandler): - def delete(self, flow_id): if self.flow.killable: self.flow.kill(self.master) @@ -246,75 +242,78 @@ class FlowHandler(RequestHandler): def put(self, flow_id): flow = self.flow flow.backup() - for a, b in self.json.items(): - if a == "request" and hasattr(flow, "request"): - request = flow.request - for k, v in b.items(): - if k in ["method", "scheme", "host", "path", "http_version"]: - setattr(request, k, str(v)) - elif k == "port": - request.port = int(v) - elif k == "headers": - request.headers.clear() - for header in v: - request.headers.add(*header) - elif k == "content": - request.text = v - else: - print("Warning: Unknown update {}.{}: {}".format(a, k, v)) - - elif a == "response" and hasattr(flow, "response"): - response = flow.response - for k, v in b.items(): - if k == "msg": - response.msg = str(v) - elif k == "code": - response.status_code = int(v) - elif k == "http_version": - response.http_version = str(v) - elif k == "headers": - response.headers.clear() - for header in v: - response.headers.add(*header) - elif k == "content": - response.text = v - else: - print("Warning: Unknown update {}.{}: {}".format(a, k, v)) - else: - print("Warning: Unknown update {}: {}".format(a, b)) + try: + for a, b in self.json.items(): + if a == "request" and hasattr(flow, "request"): + request = flow.request + for k, v in b.items(): + if k in ["method", "scheme", "host", "path", "http_version"]: + setattr(request, k, str(v)) + elif k == "port": + request.port = int(v) + elif k == "headers": + request.headers.clear() + for header in v: + request.headers.add(*header) + elif k == "content": + request.text = v + else: + raise APIError(400, "Unknown update request.{}: {}".format(k, v)) + + elif a == "response" and hasattr(flow, "response"): + response = flow.response + for k, v in b.items(): + if k in ["msg", "http_version"]: + setattr(response, k, str(v)) + elif k == "code": + response.status_code = int(v) + elif k == "headers": + response.headers.clear() + for header in v: + response.headers.add(*header) + elif k == "content": + response.text = v + else: + raise APIError(400, "Unknown update response.{}: {}".format(k, v)) + else: + raise APIError(400, "Unknown update {}: {}".format(a, b)) + except APIError: + flow.revert() + raise self.view.update(flow) class DuplicateFlow(RequestHandler): - def post(self, flow_id): - self.master.view.duplicate_flow(self.flow) + f = self.flow.copy() + self.view.add(f) + self.write(f.id) class RevertFlow(RequestHandler): - def post(self, flow_id): - self.flow.revert() + if self.flow.modified(): + self.flow.revert() + self.view.update(self.flow) class ReplayFlow(RequestHandler): - def post(self, flow_id): self.flow.backup() self.flow.response = None self.view.update(self.flow) - r = self.master.replay_request(self.flow) - if r: - raise APIError(400, r) + try: + self.master.replay_request(self.flow) + except exceptions.ReplayException as e: + raise APIError(400, str(e)) class FlowContent(RequestHandler): - def post(self, flow_id, message): self.flow.backup() message = getattr(self.flow, message) - message.content = self.request.files.values()[0][0].body + message.content = self.filecontents self.view.update(self.flow) def get(self, flow_id, message): @@ -347,15 +346,14 @@ class FlowContent(RequestHandler): class FlowContentView(RequestHandler): - def get(self, flow_id, message, content_view): message = getattr(self.flow, message) description, lines, error = contentviews.get_message_content_view( content_view.replace('_', ' '), message ) -# if error: -# add event log + # if error: + # add event log self.write(dict( lines=list(lines), @@ -364,13 +362,11 @@ class FlowContentView(RequestHandler): class Events(RequestHandler): - def get(self): - self.write([]) # FIXME + self.write([logentry_to_json(e) for e in self.master.events.data]) class Settings(RequestHandler): - def get(self): self.write(dict( version=version.VERSION, @@ -389,51 +385,20 @@ class Settings(RequestHandler): )) def put(self): - update = {} - for k, v in self.json.items(): - if k == "intercept": - self.master.options.intercept = v - update[k] = v - elif k == "showhost": - self.master.options.showhost = v - update[k] = v - elif k == "no_upstream_cert": - self.master.options.no_upstream_cert = v - update[k] = v - elif k == "rawtcp": - self.master.options.rawtcp = v - update[k] = v - elif k == "http2": - self.master.options.http2 = v - update[k] = v - elif k == "anticache": - self.master.options.anticache = v - update[k] = v - elif k == "anticomp": - self.master.options.anticomp = v - update[k] = v - elif k == "stickycookie": - self.master.options.stickycookie = v - update[k] = v - elif k == "stickyauth": - self.master.options.stickyauth = v - update[k] = v - elif k == "stream": - self.master.options.stream_large_bodies = v - update[k] = v - else: - print("Warning: Unknown setting {}: {}".format(k, v)) - - ClientConnection.broadcast( - resource="settings", - cmd="update", - data=update - ) + update = self.json + option_whitelist = { + "intercept", "showhost", "no_upstream_cert", + "rawtcp", "http2", "anticache", "anticomp", + "stickycookie", "stickyauth", "stream_large_bodies" + } + for k in update: + if k not in option_whitelist: + raise APIError(400, "Unknown setting {}".format(k)) + self.master.options.update(**update) class Application(tornado.web.Application): - - def __init__(self, master, debug, wauthenticator): + def __init__(self, master, debug): self.master = master handlers = [ (r"/", IndexHandler), @@ -449,7 +414,9 @@ class Application(tornado.web.Application): (r"/flows/(?P<flow_id>[0-9a-f\-]+)/replay", ReplayFlow), (r"/flows/(?P<flow_id>[0-9a-f\-]+)/revert", RevertFlow), (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content", FlowContent), - (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)", FlowContentView), + ( + r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)", + FlowContentView), (r"/settings", Settings), (r"/clear", ClearAll), ] @@ -460,6 +427,5 @@ class Application(tornado.web.Application): cookie_secret=os.urandom(256), debug=debug, autoreload=False, - wauthenticator=wauthenticator, ) super().__init__(handlers, **settings) diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index d2203f10..5457fb40 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,49 +1,20 @@ import sys import webbrowser +from typing import Optional import tornado.httpserver import tornado.ioloop - -from typing import Optional - from mitmproxy import addons from mitmproxy import exceptions -from mitmproxy.addons import view -from mitmproxy.addons import intercept -from mitmproxy import options +from mitmproxy import log from mitmproxy import master +from mitmproxy import options +from mitmproxy.addons import eventstore +from mitmproxy.addons import intercept +from mitmproxy.addons import view from mitmproxy.tools.web import app -class Stop(Exception): - pass - - -class _WebState(): - def add_log(self, e, level): - # server-side log ids are odd - self._last_event_id += 2 - entry = { - "id": self._last_event_id, - "message": e, - "level": level - } - self.events.append(entry) - app.ClientConnection.broadcast( - resource="events", - cmd="add", - data=entry - ) - - def clear(self): - super().clear() - self.events.clear() - app.ClientConnection.broadcast( - resource="events", - cmd="reset" - ) - - class Options(options.Options): def __init__( self, @@ -52,54 +23,34 @@ class Options(options.Options): wdebug: bool = False, wport: int = 8081, wiface: str = "127.0.0.1", - # wauthenticator: Optional[authentication.PassMan] = None, - wsingleuser: Optional[str] = None, - whtpasswd: Optional[str] = None, **kwargs ) -> None: + self.intercept = intercept self.wdebug = wdebug self.wport = wport self.wiface = wiface - # self.wauthenticator = wauthenticator - # self.wsingleuser = wsingleuser - # self.whtpasswd = whtpasswd - self.intercept = intercept super().__init__(**kwargs) - # TODO: This doesn't belong here. - def process_web_options(self, parser): - # if self.wsingleuser or self.whtpasswd: - # if self.wsingleuser: - # if len(self.wsingleuser.split(':')) != 2: - # return parser.error( - # "Invalid single-user specification. Please use the format username:password" - # ) - # username, password = self.wsingleuser.split(':') - # # self.wauthenticator = authentication.PassManSingleUser(username, password) - # elif self.whtpasswd: - # try: - # self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd) - # except ValueError as v: - # return parser.error(v.message) - # else: - # self.wauthenticator = None - pass - class WebMaster(master.Master): - def __init__(self, options, server): super().__init__(options, server) self.view = view.View() - self.view.sig_view_add.connect(self._sig_add) - self.view.sig_view_remove.connect(self._sig_remove) - self.view.sig_view_update.connect(self._sig_update) - self.view.sig_view_refresh.connect(self._sig_refresh) + self.view.sig_view_add.connect(self._sig_view_add) + self.view.sig_view_remove.connect(self._sig_view_remove) + self.view.sig_view_update.connect(self._sig_view_update) + self.view.sig_view_refresh.connect(self._sig_view_refresh) + + self.events = eventstore.EventStore() + self.events.sig_add.connect(self._sig_events_add) + self.events.sig_refresh.connect(self._sig_events_refresh) + + self.options.changed.connect(self._sig_options_update) self.addons.add(*addons.default_addons()) - self.addons.add(self.view, intercept.Intercept()) + self.addons.add(self.view, self.events, intercept.Intercept()) self.app = app.Application( - self, self.options.wdebug, False + self, self.options.wdebug ) # This line is just for type hinting self.options = self.options # type: Options @@ -112,33 +63,53 @@ class WebMaster(master.Master): "error" ) - def _sig_add(self, view, flow): + def _sig_view_add(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="add", - data=app.convert_flow_to_json_dict(flow) + data=app.flow_to_json(flow) ) - def _sig_update(self, view, flow): + def _sig_view_update(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="update", - data=app.convert_flow_to_json_dict(flow) + data=app.flow_to_json(flow) ) - def _sig_remove(self, view, flow): + def _sig_view_remove(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="remove", data=dict(id=flow.id) ) - def _sig_refresh(self, view): + def _sig_view_refresh(self, view): app.ClientConnection.broadcast( resource="flows", cmd="reset" ) + def _sig_events_add(self, event_store, entry: log.LogEntry): + app.ClientConnection.broadcast( + resource="events", + cmd="add", + data=app.logentry_to_json(entry) + ) + + def _sig_events_refresh(self, event_store): + app.ClientConnection.broadcast( + resource="events", + cmd="reset" + ) + + def _sig_options_update(self, options, updated): + app.ClientConnection.broadcast( + resource="settings", + cmd="update", + data={k: getattr(options, k) for k in updated} + ) + def run(self): # pragma: no cover iol = tornado.ioloop.IOLoop.instance() @@ -155,13 +126,9 @@ class WebMaster(master.Master): print("No webbrowser found. Please open a browser and point it to {}".format(url)) iol.start() - except (Stop, KeyboardInterrupt): + except (KeyboardInterrupt): self.shutdown() - # def add_log(self, e, level="info"): - # super().add_log(e, level) - # return self.state.add_log(e, level) - def open_browser(url: str) -> bool: """ |