diff options
29 files changed, 882 insertions, 670 deletions
diff --git a/docs/tutorials/gamecenter.rst b/docs/tutorials/gamecenter.rst index 9dce5df8..d0d73b73 100644 --- a/docs/tutorials/gamecenter.rst +++ b/docs/tutorials/gamecenter.rst @@ -51,7 +51,7 @@ The contents of the submission are particularly interesting: <key>context</key> <integer>0</integer> <key>score-value</key> - <integer>0</integer> + <integer>55</integer> <key>timestamp</key> <integer>1363515361321</integer> </dict> diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index b4d3c0ff..3974d736 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -2,11 +2,12 @@ from __future__ import absolute_import, print_function, division from mitmproxy.builtins import anticache from mitmproxy.builtins import anticomp +from mitmproxy.builtins import filestreamer from mitmproxy.builtins import stickyauth from mitmproxy.builtins import stickycookie from mitmproxy.builtins import script -from mitmproxy.builtins import stream from mitmproxy.builtins import replace +from mitmproxy.builtins import setheaders def default_addons(): @@ -16,6 +17,7 @@ def default_addons(): stickyauth.StickyAuth(), stickycookie.StickyCookie(), script.ScriptLoader(), - stream.Stream(), + filestreamer.FileStreamer(), replace.Replace(), + setheaders.SetHeaders(), ] diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py new file mode 100644 index 00000000..239630fb --- /dev/null +++ b/mitmproxy/builtins/dumper.py @@ -0,0 +1,252 @@ +from __future__ import absolute_import, print_function, division + +import itertools +import traceback + +import click + +from mitmproxy import contentviews +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import filt +from netlib import human +from netlib import strutils + + +def indent(n, text): + l = str(text).strip().splitlines() + pad = " " * n + return "\n".join(pad + i for i in l) + + +class Dumper(): + def __init__(self): + self.filter = None + self.flow_detail = None + self.outfp = None + self.showhost = None + + def echo(self, text, ident=None, **style): + if ident: + text = indent(ident, text) + click.secho(text, file=self.outfp, **style) + if self.outfp: + self.outfp.flush() + + def _echo_message(self, message): + if self.flow_detail >= 2 and hasattr(message, "headers"): + headers = "\r\n".join( + "{}: {}".format( + click.style( + strutils.bytes_to_escaped_str(k), fg="blue", bold=True + ), + click.style( + strutils.bytes_to_escaped_str(v), fg="blue" + ) + ) + for k, v in message.headers.fields + ) + self.echo(headers, ident=4) + if self.flow_detail >= 3: + try: + content = message.content + except ValueError: + content = message.get_content(strict=False) + + if content is None: + self.echo("(content missing)", ident=4) + elif content: + self.echo("") + + try: + type, lines = contentviews.get_content_view( + contentviews.get("Auto"), + content, + headers=getattr(message, "headers", None) + ) + except exceptions.ContentViewException: + s = "Content viewer failed: \n" + traceback.format_exc() + ctx.log.debug(s) + type, lines = contentviews.get_content_view( + contentviews.get("Raw"), + content, + headers=getattr(message, "headers", None) + ) + + styles = dict( + highlight=dict(bold=True), + offset=dict(fg="blue"), + header=dict(fg="green", bold=True), + text=dict(fg="green") + ) + + def colorful(line): + yield u" " # we can already indent here + for (style, text) in line: + yield click.style(text, **styles.get(style, {})) + + if self.flow_detail == 3: + lines_to_echo = itertools.islice(lines, 70) + else: + lines_to_echo = lines + + lines_to_echo = list(lines_to_echo) + + content = u"\r\n".join( + u"".join(colorful(line)) for line in lines_to_echo + ) + + self.echo(content) + if next(lines, None): + self.echo("(cut off)", ident=4, dim=True) + + if self.flow_detail >= 2: + self.echo("") + + def _echo_request_line(self, flow): + if flow.request.stickycookie: + stickycookie = click.style( + "[stickycookie] ", fg="yellow", bold=True + ) + else: + stickycookie = "" + + if flow.client_conn: + client = click.style( + strutils.escape_control_characters( + flow.client_conn.address.host + ), + bold=True + ) + elif flow.request.is_replay: + client = click.style("[replay]", fg="yellow", bold=True) + else: + client = "" + + method = flow.request.method + method_color = dict( + GET="green", + DELETE="red" + ).get(method.upper(), "magenta") + method = click.style( + strutils.escape_control_characters(method), + fg=method_color, + bold=True + ) + if self.showhost: + url = flow.request.pretty_url + else: + url = flow.request.url + url = click.style(strutils.escape_control_characters(url), bold=True) + + httpversion = "" + if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): + # We hide "normal" HTTP 1. + httpversion = " " + flow.request.http_version + + line = "{stickycookie}{client} {method} {url}{httpversion}".format( + stickycookie=stickycookie, + client=client, + method=method, + url=url, + httpversion=httpversion + ) + self.echo(line) + + def _echo_response_line(self, flow): + if flow.response.is_replay: + replay = click.style("[replay] ", fg="yellow", bold=True) + else: + replay = "" + + code = flow.response.status_code + code_color = None + if 200 <= code < 300: + code_color = "green" + elif 300 <= code < 400: + code_color = "magenta" + elif 400 <= code < 600: + code_color = "red" + code = click.style( + str(code), + fg=code_color, + bold=True, + blink=(code == 418) + ) + reason = click.style( + strutils.escape_control_characters(flow.response.reason), + fg=code_color, + bold=True + ) + + if flow.response.raw_content is None: + size = "(content missing)" + else: + size = human.pretty_size(len(flow.response.raw_content)) + size = click.style(size, bold=True) + + arrows = click.style(" <<", bold=True) + + line = "{replay} {arrows} {code} {reason} {size}".format( + replay=replay, + arrows=arrows, + code=code, + reason=reason, + size=size + ) + self.echo(line) + + def echo_flow(self, f): + if f.request: + self._echo_request_line(f) + self._echo_message(f.request) + + if f.response: + self._echo_response_line(f) + self._echo_message(f.response) + + if f.error: + self.echo(" << {}".format(f.error.msg), bold=True, fg="red") + + def match(self, f): + if self.flow_detail == 0: + return False + if not self.filt: + return True + elif f.match(self.filt): + return True + return False + + def configure(self, options): + if options.filtstr: + self.filt = filt.parse(options.filtstr) + if not self.filt: + raise exceptions.OptionsError( + "Invalid filter expression: %s" % options.filtstr + ) + else: + self.filt = None + self.flow_detail = options.flow_detail + self.outfp = options.tfile + self.showhost = options.showhost + + def response(self, f): + if self.match(f): + self.echo_flow(f) + + def error(self, f): + if self.match(f): + self.echo_flow(f) + + def tcp_message(self, f): + # FIXME: Filter should be applied here + if self.options.flow_detail == 0: + return + message = f.messages[-1] + direction = "->" if message.from_client else "<-" + self.echo("{client} {direction} tcp {direction} {server}".format( + client=repr(f.client_conn.address), + server=repr(f.server_conn.address), + direction=direction, + )) + self._echo_message(message) diff --git a/mitmproxy/builtins/stream.py b/mitmproxy/builtins/filestreamer.py index 821a71f1..97ddc7c4 100644 --- a/mitmproxy/builtins/stream.py +++ b/mitmproxy/builtins/filestreamer.py @@ -1,14 +1,14 @@ from __future__ import absolute_import, print_function, division import os.path -from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy.flow import io -class Stream: +class FileStreamer: def __init__(self): self.stream = None + self.active_flows = set() # type: Set[models.Flow] def start_stream_to_path(self, path, mode, filt): path = os.path.expanduser(path) @@ -17,6 +17,7 @@ class Stream: except IOError as v: return str(v) self.stream = io.FilteredFlowWriter(f, filt) + self.active_flows = set() def configure(self, options): # We're already streaming - stop the previous stream and restart @@ -38,17 +39,28 @@ class Stream: if err: raise exceptions.OptionsError(err) - def done(self): + def tcp_open(self, flow): if self.stream: - for flow in ctx.master.active_flows: - self.stream.add(flow) - self.stream.fo.close() - self.stream = None + self.active_flows.add(flow) def tcp_close(self, flow): if self.stream: self.stream.add(flow) + self.active_flows.discard(flow) def response(self, flow): if self.stream: self.stream.add(flow) + self.active_flows.discard(flow) + + def request(self, flow): + if self.stream: + self.active_flows.add(flow) + + def done(self): + if self.stream: + for flow in self.active_flows: + self.stream.add(flow) + self.active_flows = set([]) + self.stream.fo.close() + self.stream = None diff --git a/mitmproxy/builtins/setheaders.py b/mitmproxy/builtins/setheaders.py new file mode 100644 index 00000000..6bda3f55 --- /dev/null +++ b/mitmproxy/builtins/setheaders.py @@ -0,0 +1,39 @@ +from mitmproxy import exceptions +from mitmproxy import filt + + +class SetHeaders: + def __init__(self): + self.lst = [] + + def configure(self, options): + """ + options.setheaders is a tuple of (fpatt, header, value) + + fpatt: String specifying a filter pattern. + header: Header name. + value: Header value string + """ + for fpatt, header, value in options.setheaders: + cpatt = filt.parse(fpatt) + if not cpatt: + raise exceptions.OptionsError( + "Invalid setheader filter pattern %s" % fpatt + ) + self.lst.append((fpatt, header, value, cpatt)) + + def run(self, f, hdrs): + for _, header, value, cpatt in self.lst: + if cpatt(f): + hdrs.pop(header, None) + for _, header, value, cpatt in self.lst: + if cpatt(f): + hdrs.add(header, value) + + def request(self, flow): + if not flow.reply.acked: + self.run(flow, flow.request.headers) + + def response(self, flow): + if not flow.reply.acked: + self.run(flow, flow.response.headers) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index f15031c2..5d15e0cd 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -216,7 +216,7 @@ def save_data(path, data): if not path: return try: - with file(path, "wb") as f: + with open(path, "wb") as f: f.write(data) except IOError as v: signals.status_message.send(message=v.strerror) diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index ebbe8d21..bc523874 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -118,7 +118,7 @@ class ConnectionItem(urwid.WidgetWrap): return common.format_flow( self.flow, self.f, - hostheader = self.master.showhost, + hostheader = self.master.options.showhost, marked=self.state.flow_marked(self.flow) ) @@ -151,7 +151,7 @@ class ConnectionItem(urwid.WidgetWrap): if k == "a": self.master.start_server_playback( [i.copy() for i in self.master.state.view], - self.master.killextra, self.master.rheaders, + self.master.options.kill, self.master.rheaders, False, self.master.nopop, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, @@ -161,7 +161,7 @@ class ConnectionItem(urwid.WidgetWrap): elif k == "t": self.master.start_server_playback( [self.flow.copy()], - self.master.killextra, self.master.rheaders, + self.master.options.kill, self.master.rheaders, False, self.master.nopop, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index d13e9db0..c85a9f73 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -110,7 +110,7 @@ class FlowViewHeader(urwid.WidgetWrap): f, False, extended=True, - hostheader=self.master.showhost + hostheader=self.master.options.showhost ) signals.flow_change.connect(self.sig_flow_change) @@ -120,7 +120,7 @@ class FlowViewHeader(urwid.WidgetWrap): flow, False, extended=True, - hostheader=self.master.showhost + hostheader=self.master.options.showhost ) diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py index f304de57..87700fd7 100644 --- a/mitmproxy/console/grideditor.py +++ b/mitmproxy/console/grideditor.py @@ -396,7 +396,7 @@ class GridEditor(urwid.WidgetWrap): if p: try: p = os.path.expanduser(p) - d = file(p, "rb").read() + d = open(p, "rb").read() self.walker.set_current_value(d, unescaped) self.walker._modified() except IOError as v: diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 0ef12001..25a0b83f 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -210,10 +210,6 @@ class ConsoleMaster(flow.FlowMaster): self.options = self.options # type: Options self.options.errored.connect(self.options_error) - if options.setheaders: - for i in options.setheaders: - self.setheaders.add(*i) - r = self.set_intercept(options.intercept) if r: print("Intercept error: {}".format(r), file=sys.stderr) @@ -224,12 +220,8 @@ class ConsoleMaster(flow.FlowMaster): self.set_stream_large_bodies(options.stream_large_bodies) - self.refresh_server_playback = options.refresh_server_playback - self.anticache = options.anticache - self.killextra = options.kill self.rheaders = options.rheaders self.nopop = options.nopop - self.showhost = options.showhost self.palette = options.palette self.palette_transparent = options.palette_transparent @@ -377,7 +369,7 @@ class ConsoleMaster(flow.FlowMaster): if flows: self.start_server_playback( flows, - self.killextra, self.rheaders, + self.options.kill, self.rheaders, False, self.nopop, self.options.replay_ignore_params, self.options.replay_ignore_content, @@ -613,7 +605,7 @@ class ConsoleMaster(flow.FlowMaster): return path = os.path.expanduser(path) try: - f = file(path, "wb") + f = open(path, "wb") fw = flow.FlowWriter(f) for i in flows: fw.add(i) diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index f0cc4ef5..e1dd29ee 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -36,7 +36,7 @@ class Options(urwid.WidgetWrap): select.Option( "Header Set Patterns", "H", - lambda: master.setheaders.count(), + lambda: len(master.options.setheaders), self.setheaders ), select.Option( @@ -74,8 +74,8 @@ class Options(urwid.WidgetWrap): select.Option( "Show Host", "w", - lambda: master.showhost, - self.toggle_showhost + lambda: master.options.showhost, + master.options.toggler("showhost") ), select.Heading("Network"), @@ -97,25 +97,25 @@ class Options(urwid.WidgetWrap): "Anti-Cache", "a", lambda: master.options.anticache, - self.toggle_anticache + master.options.toggler("anticache") ), select.Option( "Anti-Compression", "o", lambda: master.options.anticomp, - self.toggle_anticomp + master.options.toggler("anticomp") ), select.Option( "Kill Extra", "x", - lambda: master.killextra, - self.toggle_killextra + lambda: master.options.kill, + master.options.toggler("kill") ), select.Option( "No Refresh", "f", - lambda: not master.refresh_server_playback, - self.toggle_refresh_server_playback + lambda: not master.options.refresh_server_playback, + master.options.toggler("refresh_server_playback") ), select.Option( "Sticky Auth", @@ -152,19 +152,19 @@ class Options(urwid.WidgetWrap): return super(self.__class__, self).keypress(size, key) def clearall(self): - self.master.killextra = False - self.master.showhost = False - self.master.refresh_server_playback = True self.master.server.config.no_upstream_cert = False - self.master.setheaders.clear() self.master.set_ignore_filter([]) self.master.set_tcp_filter([]) self.master.options.update( anticache = False, anticomp = False, + kill = False, + refresh_server_playback = True, replacements = [], scripts = [], + setheaders = [], + showhost = False, stickyauth = None, stickycookie = None ) @@ -177,41 +177,22 @@ class Options(urwid.WidgetWrap): expire = 1 ) - def toggle_anticache(self): - self.master.options.anticache = not self.master.options.anticache - - def toggle_anticomp(self): - self.master.options.anticomp = not self.master.options.anticomp - - def toggle_killextra(self): - self.master.killextra = not self.master.killextra - - def toggle_showhost(self): - self.master.showhost = not self.master.showhost - - def toggle_refresh_server_playback(self): - self.master.refresh_server_playback = not self.master.refresh_server_playback - def toggle_upstream_cert(self): self.master.server.config.no_upstream_cert = not self.master.server.config.no_upstream_cert signals.update_settings.send(self) def setheaders(self): - def _set(*args, **kwargs): - self.master.setheaders.set(*args, **kwargs) - signals.update_settings.send(self) self.master.view_grideditor( grideditor.SetHeadersEditor( self.master, - self.master.setheaders.get_specs(), - _set + self.master.options.setheaders, + self.master.options.setter("setheaders") ) ) def ignorepatterns(self): def _set(ignore): self.master.set_ignore_filter(ignore) - signals.update_settings.send(self) self.master.view_grideditor( grideditor.HostPatternEditor( self.master, @@ -221,14 +202,11 @@ class Options(urwid.WidgetWrap): ) def replacepatterns(self): - def _set(replacements): - self.master.options.replacements = replacements - signals.update_settings.send(self) self.master.view_grideditor( grideditor.ReplaceEditor( self.master, self.master.options.replacements, - _set + self.master.options.setter("replacements") ) ) diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index 1c3be19c..8f039e48 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -137,7 +137,7 @@ class StatusBar(urwid.WidgetWrap): def get_status(self): r = [] - if self.master.setheaders.count(): + if len(self.master.options.setheaders): r.append("[") r.append(("heading_key", "H")) r.append("eaders]") @@ -194,11 +194,11 @@ class StatusBar(urwid.WidgetWrap): opts.append("anticache") if self.master.options.anticomp: opts.append("anticomp") - if self.master.showhost: + if self.master.options.showhost: opts.append("showhost") - if not self.master.refresh_server_playback: + if not self.master.options.refresh_server_playback: opts.append("norefresh") - if self.master.killextra: + if self.master.options.kill: opts.append("killextra") if self.master.server.config.no_upstream_cert: opts.append("no-upstream-cert") diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 54d75e6b..070ec862 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -220,6 +220,12 @@ def handler(f): if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() + + # Reset the handled flag - it's common for us to feed the same object + # through handlers repeatedly, so we don't want this to persist across + # calls. + if message.reply.handled: + message.reply.handled = False return ret # Mark this function as a handler wrapper wrapper.__dict__["__handler"] = True diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index e7cebf99..eaa368a0 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -1,24 +1,19 @@ from __future__ import absolute_import, print_function, division -import itertools import sys -import traceback - -import click from typing import Optional # noqa import typing # noqa -from mitmproxy import contentviews +import click + from mitmproxy import controller from mitmproxy import exceptions -from mitmproxy import filt from mitmproxy import flow from mitmproxy import builtins from mitmproxy import utils -from netlib import human +from mitmproxy.builtins import dumper from netlib import tcp -from netlib import strutils class DumpError(Exception): @@ -28,9 +23,9 @@ class DumpError(Exception): class Options(flow.options.Options): def __init__( self, + keepserving=False, # type: bool filtstr=None, # type: Optional[str] flow_detail=1, # type: int - keepserving=False, # type: bool tfile=None, # type: Optional[typing.io.TextIO] **kwargs ): @@ -47,10 +42,9 @@ class DumpMaster(flow.FlowMaster): flow.FlowMaster.__init__(self, options, server, flow.State()) self.has_errored = False self.addons.add(*builtins.default_addons()) + self.addons.add(dumper.Dumper()) # This line is just for type hinting self.options = self.options # type: Options - self.o = options - self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.replay_ignore_host = options.replay_ignore_host @@ -64,15 +58,6 @@ class DumpMaster(flow.FlowMaster): "HTTP/2 is disabled. Use --no-http2 to silence this warning.", file=sys.stderr) - if options.filtstr: - self.filt = filt.parse(options.filtstr) - else: - self.filt = None - - if options.setheaders: - for i in options.setheaders: - self.setheaders.add(*i) - if options.server_replay: self.start_server_playback( self._readflow(options.server_replay), @@ -115,221 +100,21 @@ class DumpMaster(flow.FlowMaster): if level == "error": self.has_errored = True if self.options.verbosity >= utils.log_tier(level): - self.echo( + click.secho( e, + file=self.options.tfile, fg="red" if level == "error" else None, dim=(level == "debug"), err=(level == "error") ) - @staticmethod - def indent(n, text): - l = str(text).strip().splitlines() - pad = " " * n - return "\n".join(pad + i for i in l) - - def echo(self, text, indent=None, **style): - if indent: - text = self.indent(indent, text) - click.secho(text, file=self.options.tfile, **style) - - def _echo_message(self, message): - if self.options.flow_detail >= 2 and hasattr(message, "headers"): - headers = "\r\n".join( - "{}: {}".format( - click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True), - click.style(strutils.bytes_to_escaped_str(v), fg="blue")) - for k, v in message.headers.fields - ) - self.echo(headers, indent=4) - if self.options.flow_detail >= 3: - try: - content = message.content - except ValueError: - content = message.get_content(strict=False) - - if content is None: - self.echo("(content missing)", indent=4) - elif content: - self.echo("") - - try: - type, lines = contentviews.get_content_view( - contentviews.get("Auto"), - content, - headers=getattr(message, "headers", None) - ) - except exceptions.ContentViewException: - s = "Content viewer failed: \n" + traceback.format_exc() - self.add_log(s, "debug") - type, lines = contentviews.get_content_view( - contentviews.get("Raw"), - content, - headers=getattr(message, "headers", None) - ) - - styles = dict( - highlight=dict(bold=True), - offset=dict(fg="blue"), - header=dict(fg="green", bold=True), - text=dict(fg="green") - ) - - def colorful(line): - yield u" " # we can already indent here - for (style, text) in line: - yield click.style(text, **styles.get(style, {})) - - if self.options.flow_detail == 3: - lines_to_echo = itertools.islice(lines, 70) - else: - lines_to_echo = lines - - lines_to_echo = list(lines_to_echo) - - content = u"\r\n".join( - u"".join(colorful(line)) for line in lines_to_echo - ) - - self.echo(content) - if next(lines, None): - self.echo("(cut off)", indent=4, dim=True) - - if self.options.flow_detail >= 2: - self.echo("") - - def _echo_request_line(self, flow): - if flow.request.stickycookie: - stickycookie = click.style( - "[stickycookie] ", fg="yellow", bold=True - ) - else: - stickycookie = "" - - if flow.client_conn: - client = click.style(strutils.escape_control_characters(flow.client_conn.address.host), bold=True) - else: - client = click.style("[replay]", fg="yellow", bold=True) - - method = flow.request.method - method_color = dict( - GET="green", - DELETE="red" - ).get(method.upper(), "magenta") - method = click.style(strutils.escape_control_characters(method), fg=method_color, bold=True) - if self.showhost: - url = flow.request.pretty_url - else: - url = flow.request.url - url = click.style(strutils.escape_control_characters(url), bold=True) - - httpversion = "" - if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): - httpversion = " " + flow.request.http_version # We hide "normal" HTTP 1. - - line = "{stickycookie}{client} {method} {url}{httpversion}".format( - stickycookie=stickycookie, - client=client, - method=method, - url=url, - httpversion=httpversion - ) - self.echo(line) - - def _echo_response_line(self, flow): - if flow.response.is_replay: - replay = click.style("[replay] ", fg="yellow", bold=True) - else: - replay = "" - - code = flow.response.status_code - code_color = None - if 200 <= code < 300: - code_color = "green" - elif 300 <= code < 400: - code_color = "magenta" - elif 400 <= code < 600: - code_color = "red" - code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(strutils.escape_control_characters(flow.response.reason), fg=code_color, bold=True) - - if flow.response.raw_content is None: - size = "(content missing)" - else: - size = human.pretty_size(len(flow.response.raw_content)) - size = click.style(size, bold=True) - - arrows = click.style("<<", bold=True) - - line = "{replay} {arrows} {code} {reason} {size}".format( - replay=replay, - arrows=arrows, - code=code, - reason=reason, - size=size - ) - self.echo(line) - - def echo_flow(self, f): - if self.options.flow_detail == 0: - return - - if f.request: - self._echo_request_line(f) - self._echo_message(f.request) - - if f.response: - self._echo_response_line(f) - self._echo_message(f.response) - - if f.error: - self.echo(" << {}".format(f.error.msg), bold=True, fg="red") - - if self.options.tfile: - self.options.tfile.flush() - - def _process_flow(self, f): - if self.filt and not f.match(self.filt): - return - - self.echo_flow(f) - @controller.handler def request(self, f): - f = flow.FlowMaster.request(self, f) + f = super(DumpMaster, self).request(f) if f: self.state.delete_flow(f) return f - @controller.handler - def response(self, f): - f = flow.FlowMaster.response(self, f) - if f: - self._process_flow(f) - return f - - @controller.handler - def error(self, f): - flow.FlowMaster.error(self, f) - if f: - self._process_flow(f) - return f - - @controller.handler - def tcp_message(self, f): - super(DumpMaster, self).tcp_message(f) - - if self.options.flow_detail == 0: - return - message = f.messages[-1] - direction = "->" if message.from_client else "<-" - self.echo("{client} {direction} tcp {direction} {server}".format( - client=repr(f.client_conn.address), - server=repr(f.server_conn.address), - direction=direction, - )) - self._echo_message(message) - def run(self): # pragma: no cover if self.options.rfile and not self.options.keepserving: return diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index caa17528..b2ab74c6 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -4,8 +4,7 @@ from mitmproxy.flow import export, modules from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_flows_from_paths from mitmproxy.flow.master import FlowMaster from mitmproxy.flow.modules import ( - AppRegistry, ReplaceHooks, SetHeaders, StreamLargeBodies, ClientPlaybackState, - ServerPlaybackState + AppRegistry, StreamLargeBodies, ClientPlaybackState, ServerPlaybackState ) from mitmproxy.flow.state import State, FlowView from mitmproxy.flow import options @@ -16,7 +15,6 @@ __all__ = [ "export", "modules", "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", "FlowMaster", - "AppRegistry", "ReplaceHooks", "SetHeaders", "StreamLargeBodies", "ClientPlaybackState", - "ServerPlaybackState", "State", "FlowView", - "options", + "AppRegistry", "StreamLargeBodies", "ClientPlaybackState", + "ServerPlaybackState", "State", "FlowView", "options", ] diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 80949825..64a242ba 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -30,14 +30,11 @@ class FlowMaster(controller.Master): if server: self.add_server(server) self.state = state - self.active_flows = set() # type: Set[models.Flow] self.server_playback = None # type: Optional[modules.ServerPlaybackState] self.client_playback = None # type: Optional[modules.ClientPlaybackState] self.kill_nonreplay = False self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] - self.refresh_server_playback = False - self.setheaders = modules.SetHeaders() self.replay_ignore_params = False self.replay_ignore_content = None self.replay_ignore_host = False @@ -122,7 +119,7 @@ class FlowMaster(controller.Master): return None response = rflow.response.copy() response.is_replay = True - if self.refresh_server_playback: + if self.options.refresh_server_playback: response.refresh() flow.response = response return True @@ -329,9 +326,6 @@ class FlowMaster(controller.Master): return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) - self.active_flows.add(f) - if not f.reply.acked: - self.setheaders.run(f) if not f.reply.acked: self.process_new_request(f) return f @@ -348,11 +342,8 @@ class FlowMaster(controller.Master): @controller.handler def response(self, f): - self.active_flows.discard(f) self.state.update_flow(f) if not f.reply.acked: - self.setheaders.run(f) - if not f.reply.acked: if self.client_playback: self.client_playback.clear(f) return f @@ -367,7 +358,7 @@ class FlowMaster(controller.Master): def tcp_open(self, flow): # TODO: This would break mitmproxy currently. # self.state.add_flow(flow) - self.active_flows.add(flow) + pass @controller.handler def tcp_message(self, flow): @@ -382,4 +373,4 @@ class FlowMaster(controller.Master): @controller.handler def tcp_close(self, flow): - self.active_flows.discard(flow) + pass diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index d1f3dd42..fb3c52da 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,12 +1,10 @@ from __future__ import absolute_import, print_function, division import hashlib -import re from six.moves import urllib from mitmproxy import controller -from mitmproxy import filt from netlib import wsgi from netlib import version from netlib import strutils @@ -40,112 +38,6 @@ class AppRegistry: return self.apps.get((host, request.port), None) -class ReplaceHooks: - def __init__(self): - self.lst = [] - - def set(self, r): - self.clear() - for i in r: - self.add(*i) - - def add(self, fpatt, rex, s): - """ - add a replacement hook. - - fpatt: a string specifying a filter pattern. - rex: a regular expression. - s: the replacement string - - returns true if hook was added, false if the pattern could not be - parsed. - """ - cpatt = filt.parse(fpatt) - if not cpatt: - return False - try: - re.compile(rex) - except re.error: - return False - self.lst.append((fpatt, rex, s, cpatt)) - return True - - def get_specs(self): - """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) - tuples. - """ - return [i[:3] for i in self.lst] - - def count(self): - return len(self.lst) - - def run(self, f): - for _, rex, s, cpatt in self.lst: - if cpatt(f): - if f.response: - f.response.replace(rex, s) - else: - f.request.replace(rex, s) - - def clear(self): - self.lst = [] - - -class SetHeaders: - def __init__(self): - self.lst = [] - - def set(self, r): - self.clear() - for i in r: - self.add(*i) - - def add(self, fpatt, header, value): - """ - Add a set header hook. - - fpatt: String specifying a filter pattern. - header: Header name. - value: Header value string - - Returns True if hook was added, False if the pattern could not be - parsed. - """ - cpatt = filt.parse(fpatt) - if not cpatt: - return False - self.lst.append((fpatt, header, value, cpatt)) - return True - - def get_specs(self): - """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) - tuples. - """ - return [i[:3] for i in self.lst] - - def count(self): - return len(self.lst) - - def clear(self): - self.lst = [] - - def run(self, f): - for _, header, value, cpatt in self.lst: - if cpatt(f): - if f.response: - f.response.headers.pop(header, None) - else: - f.request.headers.pop(header, None) - for _, header, value, cpatt in self.lst: - if cpatt(f): - if f.response: - f.response.headers.add(header, value) - else: - f.request.headers.add(header, value) - - class StreamLargeBodies(object): def __init__(self, max_size): self.max_size = max_size diff --git a/mitmproxy/options.py b/mitmproxy/options.py index a124eaf6..04353dca 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -76,10 +76,23 @@ class Options(object): self.changed.send(self) def setter(self, attr): + """ + Generate a setter for a given attribute. This returns a callable + taking a single argument. + """ if attr not in self._opts: raise KeyError("No such option: %s" % attr) return lambda x: self.__setattr__(attr, x) + def toggler(self, attr): + """ + Generate a toggler for a boolean attribute. This returns a callable + that takes no arguments. + """ + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + return lambda: self.__setattr__(attr, not getattr(self, attr)) + def __repr__(self): options = pprint.pformat(self._opts, indent=4).strip(" {}") if "\n" in options: diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 27c2a664..ee66393f 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -130,6 +130,7 @@ class Http2Layer(base.Layer): [repr(event)] ) + eid = None if hasattr(event, 'stream_id'): if is_server and event.stream_id % 2 == 1: eid = self.server_to_client_stream_ids[event.stream_id] @@ -137,83 +138,124 @@ class Http2Layer(base.Layer): eid = event.stream_id if isinstance(event, events.RequestReceived): - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) - self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) - self.streams[eid].timestamp_start = time.time() - self.streams[eid].no_body = (event.stream_ended is not None) - if event.priority_updated is not None: - self.streams[eid].priority_exclusive = event.priority_updated.exclusive - self.streams[eid].priority_depends_on = event.priority_updated.depends_on - self.streams[eid].priority_weight = event.priority_updated.weight - self.streams[eid].handled_priority_event = event.priority_updated - self.streams[eid].start() + return self._handle_request_received(eid, event) elif isinstance(event, events.ResponseReceived): - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) - self.streams[eid].queued_data_length = 0 - self.streams[eid].timestamp_start = time.time() - self.streams[eid].response_headers = headers - self.streams[eid].response_arrived.set() + return self._handle_response_received(eid, event) elif isinstance(event, events.DataReceived): - if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit: - self.streams[eid].zombie = time.time() - source_conn.h2.safe_reset_stream(event.stream_id, 0x7) - self.log("HTTP body too large. Limit is {}.".format(self.config.body_size_limit), "info") - else: - self.streams[eid].data_queue.put(event.data) - self.streams[eid].queued_data_length += len(event.data) - source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) + return self._handle_data_received(eid, event, source_conn) elif isinstance(event, events.StreamEnded): - self.streams[eid].timestamp_end = time.time() - self.streams[eid].data_finished.set() + return self._handle_stream_ended(eid) elif isinstance(event, events.StreamReset): - self.streams[eid].zombie = time.time() - if eid in self.streams and event.error_code == 0x8: - if is_server: - other_stream_id = self.streams[eid].client_stream_id - else: - other_stream_id = self.streams[eid].server_stream_id - if other_stream_id is not None: - other_conn.h2.safe_reset_stream(other_stream_id, event.error_code) + return self._handle_stream_reset(eid, event, is_server, other_conn) elif isinstance(event, events.RemoteSettingsChanged): - new_settings = dict([(id, cs.new_value) for (id, cs) in six.iteritems(event.changed_settings)]) - other_conn.h2.safe_update_settings(new_settings) + return self._handle_remote_settings_changed(event, other_conn) elif isinstance(event, events.ConnectionTerminated): - if event.error_code == h2.errors.NO_ERROR: - # Do not immediately terminate the other connection. - # Some streams might be still sending data to the client. - return False - else: - # Something terrible has happened - kill everything! - self.client_conn.h2.close_connection( - error_code=event.error_code, - last_stream_id=event.last_stream_id, - additional_data=event.additional_data - ) - self.client_conn.send(self.client_conn.h2.data_to_send()) - self._kill_all_streams() - return False + return self._handle_connection_terminated(event) elif isinstance(event, events.PushedStreamReceived): - # pushed stream ids should be unique and not dependent on race conditions - # only the parent stream id must be looked up first - parent_eid = self.server_to_client_stream_ids[event.parent_stream_id] - with self.client_conn.h2.lock: - self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) - self.client_conn.send(self.client_conn.h2.data_to_send()) - - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) - self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, event.pushed_stream_id, headers) - self.streams[event.pushed_stream_id].timestamp_start = time.time() - self.streams[event.pushed_stream_id].pushed = True - self.streams[event.pushed_stream_id].parent_stream_id = parent_eid - self.streams[event.pushed_stream_id].timestamp_end = time.time() - self.streams[event.pushed_stream_id].request_data_finished.set() - self.streams[event.pushed_stream_id].start() + return self._handle_pushed_stream_received(event) elif isinstance(event, events.PriorityUpdated): - if eid in self.streams and self.streams[eid].handled_priority_event is event: - # this event was already handled during stream creation - # HeadersFrame + Priority information as RequestReceived - return True + return self._handle_priority_updated(eid, event) + elif isinstance(event, events.TrailersReceived): + raise NotImplementedError('TrailersReceived not implemented') + + # fail-safe for unhandled events + return True + + def _handle_request_received(self, eid, event): + headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) + self.streams[eid].timestamp_start = time.time() + self.streams[eid].no_body = (event.stream_ended is not None) + if event.priority_updated is not None: + self.streams[eid].priority_exclusive = event.priority_updated.exclusive + self.streams[eid].priority_depends_on = event.priority_updated.depends_on + self.streams[eid].priority_weight = event.priority_updated.weight + self.streams[eid].handled_priority_event = event.priority_updated + self.streams[eid].start() + return True + + def _handle_response_received(self, eid, event): + headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + self.streams[eid].queued_data_length = 0 + self.streams[eid].timestamp_start = time.time() + self.streams[eid].response_headers = headers + self.streams[eid].response_arrived.set() + return True + + def _handle_data_received(self, eid, event, source_conn): + if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit: + self.streams[eid].zombie = time.time() + source_conn.h2.safe_reset_stream(event.stream_id, h2.errors.REFUSED_STREAM) + self.log("HTTP body too large. Limit is {}.".format(self.config.body_size_limit), "info") + else: + self.streams[eid].data_queue.put(event.data) + self.streams[eid].queued_data_length += len(event.data) + source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) + return True + + def _handle_stream_ended(self, eid): + self.streams[eid].timestamp_end = time.time() + self.streams[eid].data_finished.set() + return True + + def _handle_stream_reset(self, eid, event, is_server, other_conn): + self.streams[eid].zombie = time.time() + if eid in self.streams and event.error_code == h2.errors.CANCEL: + if is_server: + other_stream_id = self.streams[eid].client_stream_id + else: + other_stream_id = self.streams[eid].server_stream_id + if other_stream_id is not None: + other_conn.h2.safe_reset_stream(other_stream_id, event.error_code) + return True + + def _handle_remote_settings_changed(self, event, other_conn): + new_settings = dict([(key, cs.new_value) for (key, cs) in six.iteritems(event.changed_settings)]) + other_conn.h2.safe_update_settings(new_settings) + return True + + def _handle_connection_terminated(self, event): + if event.error_code != h2.errors.NO_ERROR: + # Something terrible has happened - kill everything! + self.client_conn.h2.close_connection( + error_code=event.error_code, + last_stream_id=event.last_stream_id, + additional_data=event.additional_data + ) + self.client_conn.send(self.client_conn.h2.data_to_send()) + self._kill_all_streams() + else: + """ + Do not immediately terminate the other connection. + Some streams might be still sending data to the client. + """ + return False + + def _handle_pushed_stream_received(self, event): + # pushed stream ids should be unique and not dependent on race conditions + # only the parent stream id must be looked up first + parent_eid = self.server_to_client_stream_ids[event.parent_stream_id] + with self.client_conn.h2.lock: + self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) + self.client_conn.send(self.client_conn.h2.data_to_send()) + + headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, event.pushed_stream_id, headers) + self.streams[event.pushed_stream_id].timestamp_start = time.time() + self.streams[event.pushed_stream_id].pushed = True + self.streams[event.pushed_stream_id].parent_stream_id = parent_eid + self.streams[event.pushed_stream_id].timestamp_end = time.time() + self.streams[event.pushed_stream_id].request_data_finished.set() + self.streams[event.pushed_stream_id].start() + return True + + def _handle_priority_updated(self, eid, event): + if eid in self.streams and self.streams[eid].handled_priority_event is event: + # this event was already handled during stream creation + # HeadersFrame + Priority information as RequestReceived + return True + with self.server_conn.h2.lock: mapped_stream_id = event.stream_id if mapped_stream_id in self.streams and self.streams[mapped_stream_id].server_stream_id: # if the stream is already up and running and was sent to the server @@ -225,17 +267,13 @@ class Http2Layer(base.Layer): self.streams[eid].priority_depends_on = event.depends_on self.streams[eid].priority_weight = event.weight - with self.server_conn.h2.lock: - self.server_conn.h2.prioritize( - mapped_stream_id, - weight=event.weight, - depends_on=self._map_depends_on_stream_id(mapped_stream_id, event.depends_on), - exclusive=event.exclusive - ) - self.server_conn.send(self.server_conn.h2.data_to_send()) - elif isinstance(event, events.TrailersReceived): - raise NotImplementedError("TrailersReceived not implemented") - + self.server_conn.h2.prioritize( + mapped_stream_id, + weight=event.weight, + depends_on=self._map_depends_on_stream_id(mapped_stream_id, event.depends_on), + exclusive=event.exclusive + ) + self.server_conn.send(self.server_conn.h2.data_to_send()) return True def _map_depends_on_stream_id(self, stream_id, depends_on): @@ -337,6 +375,15 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.priority_weight = None self.handled_priority_event = None + def check_close_connection(self, flow): + # This layer only handles a single stream. + # RFC 7540 8.1: An HTTP request/response exchange fully consumes a single stream. + return True + + def set_server(self, *args, **kwargs): # pragma: no cover + # do not mess with the server connection - all streams share it. + pass + @property def data_queue(self): if self.response_arrived.is_set(): @@ -428,15 +475,25 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) headers.insert(0, ":method", message.method) headers.insert(0, ":scheme", message.scheme) + priority_exclusive = None + priority_depends_on = None + priority_weight = None + if self.handled_priority_event: + # only send priority information if they actually came with the original HeadersFrame + # and not if they got updated before/after with a PriorityFrame + priority_exclusive = self.priority_exclusive + priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) + priority_weight = self.priority_weight + try: self.server_conn.h2.safe_send_headers( self.is_zombie, self.server_stream_id, headers, end_stream=self.no_body, - priority_exclusive=self.priority_exclusive, - priority_depends_on=self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on), - priority_weight=self.priority_weight, + priority_exclusive=priority_exclusive, + priority_depends_on=priority_depends_on, + priority_weight=priority_weight, ) except Exception as e: # pragma: no cover raise e @@ -477,7 +534,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) while True: try: yield self.response_data_queue.get(timeout=1) - except queue.Empty: + except queue.Empty: # pragma: no cover pass if self.response_data_finished.is_set(): if self.zombie: # pragma: no cover @@ -512,19 +569,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) if self.zombie: # pragma: no cover raise exceptions.Http2ProtocolException("Zombie Stream") - def check_close_connection(self, flow): - # This layer only handles a single stream. - # RFC 7540 8.1: An HTTP request/response exchange fully consumes a single stream. - return True - - def set_server(self, *args, **kwargs): # pragma: no cover - # do not mess with the server connection - all streams share it. - pass - def run(self): - self() - - def __call__(self): layer = http.HttpLayer(self, self.mode) try: diff --git a/netlib/http/http1/read.py b/netlib/http/http1/read.py index a4c341fd..70fffbd4 100644 --- a/netlib/http/http1/read.py +++ b/netlib/http/http1/read.py @@ -244,7 +244,7 @@ def _read_request_line(rfile): raise exceptions.HttpReadDisconnect("Client disconnected") try: - method, path, http_version = line.split(b" ") + method, path, http_version = line.split() if path == b"*" or path.startswith(b"/"): form = "relative" @@ -291,8 +291,7 @@ def _read_response_line(rfile): raise exceptions.HttpReadDisconnect("Server disconnected") try: - - parts = line.split(b" ", 2) + parts = line.split(None, 2) if len(parts) == 2: # handle missing message gracefully parts.append(b"") diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py new file mode 100644 index 00000000..57e3d036 --- /dev/null +++ b/test/mitmproxy/builtins/test_dumper.py @@ -0,0 +1,86 @@ +from .. import tutils, mastertest +from six.moves import cStringIO as StringIO + +from mitmproxy.builtins import dumper +from mitmproxy.flow import state +from mitmproxy import exceptions +from mitmproxy import dump +from mitmproxy import models +import netlib.tutils +import mock + + +class TestDumper(mastertest.MasterTest): + def test_simple(self): + d = dumper.Dumper() + sio = StringIO() + + d.configure(dump.Options(tfile = sio, flow_detail = 0)) + d.response(tutils.tflow()) + assert not sio.getvalue() + + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow()) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(resp=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(err=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow() + flow.request = netlib.tutils.treq() + flow.request.stickycookie = True + flow.client_conn = mock.MagicMock() + flow.client_conn.address.host = "foo" + flow.response = netlib.tutils.tresp(content=None) + flow.response.is_replay = True + flow.response.status_code = 300 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) + flow.response.headers["content-type"] = "application/json" + flow.response.status_code = 400 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio)) + flow = tutils.tflow() + flow.request.content = None + flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) + flow.response.content = None + d.response(flow) + assert "content missing" in sio.getvalue() + + +class TestContentView(mastertest.MasterTest): + @mock.patch("mitmproxy.contentviews.get_content_view") + def test_contentview(self, get_content_view): + se = exceptions.ContentViewException(""), ("x", iter([])) + get_content_view.side_effect = se + + s = state.State() + sio = StringIO() + m = mastertest.RecordingMaster( + dump.Options( + flow_detail=4, + verbosity=3, + tfile=sio, + ), + None, s + ) + d = dumper.Dumper() + m.addons.add(d) + self.invoke(m, "response", tutils.tflow()) + assert "Content viewer failed" in m.event_log[0][1] diff --git a/test/mitmproxy/builtins/test_stream.py b/test/mitmproxy/builtins/test_filestreamer.py index edaa41d2..002006b7 100644 --- a/test/mitmproxy/builtins/test_stream.py +++ b/test/mitmproxy/builtins/test_filestreamer.py @@ -4,7 +4,7 @@ from .. import tutils, mastertest import os.path -from mitmproxy.builtins import stream +from mitmproxy.builtins import filestreamer from mitmproxy.flow import master, FlowReader from mitmproxy.flow import state from mitmproxy.flow import options @@ -27,7 +27,7 @@ class TestStream(mastertest.MasterTest): None, s ) - sa = stream.Stream() + sa = filestreamer.FileStreamer() m.addons.add(sa) f = tutils.tflow(resp=True) diff --git a/test/mitmproxy/builtins/test_setheaders.py b/test/mitmproxy/builtins/test_setheaders.py new file mode 100644 index 00000000..1a8d048c --- /dev/null +++ b/test/mitmproxy/builtins/test_setheaders.py @@ -0,0 +1,64 @@ +from .. import tutils, mastertest + +from mitmproxy.builtins import setheaders +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestSetHeaders(mastertest.MasterTest): + def mkmaster(self, **opts): + s = state.State() + m = mastertest.RecordingMaster(options.Options(**opts), None, s) + sh = setheaders.SetHeaders() + m.addons.add(sh) + return m, sh + + def test_configure(self): + sh = setheaders.SetHeaders() + tutils.raises( + "invalid setheader filter pattern", + sh.configure, + options.Options( + setheaders = [("~b", "one", "two")] + ) + ) + + def test_setheaders(self): + m, sh = self.mkmaster( + setheaders = [ + ("~q", "one", "two"), + ("~s", "one", "three") + ] + ) + f = tutils.tflow() + f.request.headers["one"] = "xxx" + self.invoke(m, "request", f) + assert f.request.headers["one"] == "two" + + f = tutils.tflow(resp=True) + f.response.headers["one"] = "xxx" + self.invoke(m, "response", f) + assert f.response.headers["one"] == "three" + + m, sh = self.mkmaster( + setheaders = [ + ("~s", "one", "two"), + ("~s", "one", "three") + ] + ) + f = tutils.tflow(resp=True) + f.request.headers["one"] = "xxx" + f.response.headers["one"] = "xxx" + self.invoke(m, "response", f) + assert f.response.headers.get_all("one") == ["two", "three"] + + m, sh = self.mkmaster( + setheaders = [ + ("~q", "one", "two"), + ("~q", "one", "three") + ] + ) + f = tutils.tflow() + f.request.headers["one"] = "xxx" + self.invoke(m, "request", f) + assert f.request.headers.get_all("one") == ["two", "three"] diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index d1fe8cb4..dcc0dc48 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -8,11 +8,12 @@ from mitmproxy import flow, proxy, models, controller class MasterTest: - def invoke(self, master, handler, message): + def invoke(self, master, handler, *message): with master.handlecontext(): func = getattr(master, handler) - func(message) - message.reply = controller.DummyReply() + func(*message) + if message: + message[0].reply = controller.DummyReply() def cycle(self, master, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index c94630a9..90f33264 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -1,67 +1,11 @@ import os from six.moves import cStringIO as StringIO -from mitmproxy.exceptions import ContentViewException -import netlib.tutils - -from mitmproxy import dump, flow, models, exceptions +from mitmproxy import dump, flow, exceptions from . import tutils, mastertest import mock -def test_strfuncs(): - o = dump.Options( - tfile = StringIO(), - flow_detail = 0, - ) - m = dump.DumpMaster(None, o) - - m.o.flow_detail = 0 - m.echo_flow(tutils.tflow()) - assert not o.tfile.getvalue() - - m.o.flow_detail = 4 - m.echo_flow(tutils.tflow()) - assert o.tfile.getvalue() - - o.tfile = StringIO() - m.echo_flow(tutils.tflow(resp=True)) - assert "<<" in o.tfile.getvalue() - - o.tfile = StringIO() - m.echo_flow(tutils.tflow(err=True)) - assert "<<" in o.tfile.getvalue() - - flow = tutils.tflow() - flow.request = netlib.tutils.treq() - flow.request.stickycookie = True - flow.client_conn = mock.MagicMock() - flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=None) - flow.response.is_replay = True - flow.response.status_code = 300 - m.echo_flow(flow) - - flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) - flow.response.headers["content-type"] = "application/json" - flow.response.status_code = 400 - m.echo_flow(flow) - - -@mock.patch("mitmproxy.contentviews.get_content_view") -def test_contentview(get_content_view): - get_content_view.side_effect = ContentViewException(""), ("x", iter([])) - - o = dump.Options( - flow_detail=4, - verbosity=3, - tfile=StringIO(), - ) - m = dump.DumpMaster(None, o) - m.echo_flow(tutils.tflow()) - assert "Content viewer failed" in m.options.tfile.getvalue() - - class TestDumpMaster(mastertest.MasterTest): def dummy_cycle(self, master, n, content): mastertest.MasterTest.dummy_cycle(self, master, n, content) @@ -72,11 +16,7 @@ class TestDumpMaster(mastertest.MasterTest): options["verbosity"] = 0 if "flow_detail" not in options: options["flow_detail"] = 0 - o = dump.Options( - filtstr=filt, - tfile=StringIO(), - **options - ) + o = dump.Options(filtstr=filt, tfile=StringIO(), **options) return dump.DumpMaster(None, o) def test_basic(self): @@ -104,24 +44,10 @@ class TestDumpMaster(mastertest.MasterTest): ) m = dump.DumpMaster(None, o) f = tutils.tflow(err=True) - m.request(f) + m.error(f) assert m.error(f) assert "error" in o.tfile.getvalue() - def test_missing_content(self): - o = dump.Options( - flow_detail=3, - tfile=StringIO(), - ) - m = dump.DumpMaster(None, o) - f = tutils.tflow() - f.request.content = None - m.request(f) - f.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) - f.response.content = None - m.response(f) - assert "content missing" in o.tfile.getvalue() - def test_replay(self): o = dump.Options(server_replay=["nonexistent"], kill=True) tutils.raises(dump.DumpError, dump.DumpMaster, None, o) @@ -155,9 +81,8 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) assert "GET" in self.dummy_cycle( self.mkmaster(None, flow_detail=1, rfile=p), - 0, b"", + 1, b"", ) - tutils.raises( dump.DumpError, self.mkmaster, None, verbosity=1, rfile="/nonexistent" diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 10163401..90f7f915 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -751,7 +751,11 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(None, DummyServer(ProxyConfig()), s) + fm = flow.FlowMaster( + flow.options.Options(), + DummyServer(ProxyConfig()), + s + ) assert not fm.start_server_playback( pb, False, @@ -779,7 +783,7 @@ class TestFlowMaster: f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, None, s) + fm = flow.FlowMaster(flow.options.Options(), None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) @@ -961,55 +965,3 @@ class TestClientConnection: assert c3.get_state() == c.get_state() assert str(c) - - -def test_setheaders(): - h = flow.SetHeaders() - h.add("~q", "foo", "bar") - assert h.lst - - h.set( - [ - (".*", "one", "two"), - (".*", "three", "four"), - ] - ) - assert h.count() == 2 - - h.clear() - assert not h.lst - - h.add("~q", "foo", "bar") - h.add("~s", "foo", "bar") - - v = h.get_specs() - assert v == [('~q', 'foo', 'bar'), ('~s', 'foo', 'bar')] - assert h.count() == 2 - h.clear() - assert h.count() == 0 - - f = tutils.tflow() - f.request.content = b"foo" - h.add("~s", "foo", "bar") - h.run(f) - assert f.request.content == b"foo" - - h.clear() - h.add("~s", "one", "two") - h.add("~s", "one", "three") - f = tutils.tflow(resp=True) - f.request.headers["one"] = "xxx" - f.response.headers["one"] = "xxx" - h.run(f) - assert f.request.headers["one"] == "xxx" - assert f.response.headers.get_all("one") == ["two", "three"] - - h.clear() - h.add("~q", "one", "two") - h.add("~q", "one", "three") - f = tutils.tflow() - f.request.headers["one"] = "xxx" - h.run(f) - assert f.request.headers.get_all("one") == ["two", "three"] - - assert not h.add("~", "foo", "bar") diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py index cdb0d765..af619b27 100644 --- a/test/mitmproxy/test_options.py +++ b/test/mitmproxy/test_options.py @@ -52,6 +52,17 @@ def test_setter(): o.setter("nonexistent") +def test_toggler(): + o = TO(two=True) + f = o.toggler("two") + f() + assert o.two is False + f() + assert o.two is True + with tutils.raises("no such option"): + o.toggler("nonexistent") + + def test_rollback(): o = TO(one="two") diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index a100ac2d..b8f724bd 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -87,31 +87,31 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): class _Http2TestBase(object): @classmethod - def setup_class(self): - self.config = ProxyConfig(**self.get_proxy_config()) + def setup_class(cls): + cls.config = ProxyConfig(**cls.get_proxy_config()) - tmaster = tservers.TestMaster(self.config) + tmaster = tservers.TestMaster(cls.config) tmaster.start_app(APP_HOST, APP_PORT) - self.proxy = tservers.ProxyThread(tmaster) - self.proxy.start() + cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy.start() @classmethod def teardown_class(cls): cls.proxy.shutdown() - @property - def master(self): - return self.proxy.tmaster - @classmethod def get_proxy_config(cls): cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") return dict( - no_upstream_cert = False, - cadir = cls.cadir, - authenticator = None, + no_upstream_cert=False, + cadir=cls.cadir, + authenticator=None, ) + @property + def master(self): + return self.proxy.tmaster + def setup(self): self.master.clear_log() self.master.state.clear() @@ -150,13 +150,17 @@ class _Http2TestBase(object): stream_id=1, headers=[], body=b'', + end_stream=None, priority_exclusive=None, priority_depends_on=None, priority_weight=None): + if end_stream is None: + end_stream = (len(body) == 0) + h2_conn.send_headers( stream_id=stream_id, headers=headers, - end_stream=(len(body) == 0), + end_stream=end_stream, priority_exclusive=priority_exclusive, priority_depends_on=priority_depends_on, priority_weight=priority_weight, @@ -376,6 +380,153 @@ class TestRequestWithPriority(_Http2Test): @requires_alpn +class TestPriority(_Http2Test): + priority_data = None + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.PriorityUpdated): + self.priority_data = (event.exclusive, event.depends_on, event.weight) + elif isinstance(event, h2.events.RequestReceived): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + h2_conn.send_headers(event.stream_id, headers) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_priority(self): + client, h2_conn = self._setup_connection() + + h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.priority_data == (True, 0, 42) + + +@requires_alpn +class TestPriorityWithExistingStream(_Http2Test): + priority_data = [] + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.PriorityUpdated): + self.priority_data.append((event.exclusive, event.depends_on, event.weight)) + elif isinstance(event, h2.events.RequestReceived): + assert not event.priority_updated + + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + h2_conn.send_headers(event.stream_id, headers) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + elif isinstance(event, h2.events.StreamEnded): + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_priority_with_existing_stream(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + end_stream=False, + ) + + h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) + h2_conn.end_stream(1) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.priority_data == [(True, 0, 42)] + + +@requires_alpn class TestStreamResetFromServer(_Http2Test): @classmethod diff --git a/test/netlib/http/http1/test_read.py b/test/netlib/http/http1/test_read.py index 5285ac1d..c8a40ecb 100644 --- a/test/netlib/http/http1/test_read.py +++ b/test/netlib/http/http1/test_read.py @@ -1,6 +1,9 @@ from __future__ import absolute_import, print_function, division + from io import BytesIO from mock import Mock +import pytest + from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect, TcpDisconnect from netlib.http import Headers from netlib.http.http1.read import ( @@ -23,11 +26,18 @@ def test_get_header_tokens(): assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"] -def test_read_request(): - rfile = BytesIO(b"GET / HTTP/1.1\r\n\r\nskip") +@pytest.mark.parametrize("input", [ + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1 \r\n\r\nskip", +]) +def test_read_request(input): + rfile = BytesIO(input) r = read_request(rfile) assert r.method == "GET" assert r.content == b"" + assert r.http_version == "HTTP/1.1" assert r.timestamp_end assert rfile.read() == b"skip" @@ -50,11 +60,19 @@ def test_read_request_head(): assert rfile.read() == b"skip" -def test_read_response(): +@pytest.mark.parametrize("input", [ + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot \r\n\r\nbody", +]) +def test_read_response(input): req = treq() - rfile = BytesIO(b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody") + rfile = BytesIO(input) r = read_response(rfile, req) + assert r.http_version == "HTTP/1.1" assert r.status_code == 418 + assert r.reason == "I'm a teapot" assert r.content == b"body" assert r.timestamp_end |