diff options
| -rw-r--r-- | examples/__init__.py | 0 | ||||
| -rw-r--r-- | examples/har_extractor.py | 139 | ||||
| -rw-r--r-- | mitmproxy/cmdline.py | 19 | ||||
| -rw-r--r-- | mitmproxy/flow_export.py | 28 | ||||
| -rw-r--r-- | mitmproxy/models/http.py | 3 | ||||
| -rw-r--r-- | mitmproxy/protocol/http.py | 3 | ||||
| -rw-r--r-- | mitmproxy/proxy/config.py | 7 | ||||
| -rw-r--r-- | test/mitmproxy/data/har_extractor.har | 78 | ||||
| -rw-r--r-- | test/mitmproxy/test_cmdline.py | 11 | ||||
| -rw-r--r-- | test/mitmproxy/test_flow_export.py | 31 | ||||
| -rw-r--r-- | test/mitmproxy/test_har_extractor.py | 37 | ||||
| -rw-r--r-- | test/mitmproxy/test_proxy.py | 4 | ||||
| -rw-r--r-- | web/package.json | 3 | ||||
| -rw-r--r-- | web/src/js/tests/utils.js | 3 |
14 files changed, 286 insertions, 80 deletions
diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/examples/__init__.py diff --git a/examples/har_extractor.py b/examples/har_extractor.py index e7718fe8..25661f7c 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -1,5 +1,4 @@ """ - This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser to generate a HAR log object. """ @@ -17,7 +16,7 @@ class _HARLog(HAR.log): __page_count__ = 0 __page_ref__ = {} - def __init__(self, page_list): + def __init__(self, page_list=[]): self.__page_list__ = page_list self.__page_count__ = 0 self.__page_ref__ = {} @@ -67,7 +66,7 @@ def start(context, argv): '(- will output to stdout, filenames ending with .zhar ' 'will result in compressed har)' ) - context.HARLog = _HARLog(['https://github.com']) + context.HARLog = _HARLog() context.seen_server = set() @@ -83,17 +82,17 @@ def response(context, flow): # Calculate the connect_time for this server_conn. Afterwards add it to # seen list, in order to avoid the connect_time being present in entries # that use an existing connection. - connect_time = flow.server_conn.timestamp_tcp_setup - \ - flow.server_conn.timestamp_start + connect_time = (flow.server_conn.timestamp_tcp_setup - + flow.server_conn.timestamp_start) context.seen_server.add(flow.server_conn) if flow.server_conn.timestamp_ssl_setup is not None: # Get the ssl_time for this server_conn as the difference between # the start of the successful tcp setup and the successful ssl - # setup. If no ssl setup has been made it is left as -1 since it + # setup. If no ssl setup has been made it is left as -1 since it # doesn't apply to this connection. - ssl_time = flow.server_conn.timestamp_ssl_setup - \ - flow.server_conn.timestamp_tcp_setup + ssl_time = (flow.server_conn.timestamp_ssl_setup - + flow.server_conn.timestamp_tcp_setup) # Calculate the raw timings from the different timestamps present in the # request and response object. For lack of a way to measure it dns timings @@ -112,80 +111,58 @@ def response(context, flow): # HAR timings are integers in ms, so we have to re-encode the raw timings to # that format. - timings = dict([(key, int(1000 * value)) - for key, value in timings_raw.iteritems()]) + timings = dict([(k, int(1000 * v)) for k, v in timings_raw.iteritems()]) - # The full_time is the sum of all timings. Timings set to -1 will be ignored - # as per spec. - full_time = 0 - for item in timings.values(): - if item > -1: - full_time += item + # The full_time is the sum of all timings. + # Timings set to -1 will be ignored as per spec. + full_time = sum(v for v in timings.values() if v > -1) - started_date_time = datetime.fromtimestamp( - flow.request.timestamp_start, - tz=utc).isoformat() + started_date_time = datetime.utcfromtimestamp( + flow.request.timestamp_start).isoformat() request_query_string = [{"name": k, "value": v} - for k, v in flow.request.query] - request_http_version = flow.request.http_version - # Cookies are shaped as tuples by MITMProxy. - request_cookies = [{"name": k.strip(), "value": v[0]} - for k, v in flow.request.cookies.items()] - request_headers = [{"name": k, "value": v} for k, v in flow.request.headers] - request_headers_size = len(str(flow.request.headers)) - request_body_size = len(flow.request.content) - - response_http_version = flow.response.http_version - # Cookies are shaped as tuples by MITMProxy. - response_cookies = [{"name": k.strip(), "value": v[0]} - for k, v in flow.response.cookies.items()] - response_headers = [{"name": k, "value": v} - for k, v in flow.response.headers] - response_headers_size = len(str(flow.response.headers)) + for k, v in flow.request.query or {}] + response_body_size = len(flow.response.content) response_body_decoded_size = len(flow.response.get_decoded_content()) response_body_compression = response_body_decoded_size - response_body_size - response_mime_type = flow.response.headers.get('Content-Type', '') - response_redirect_url = flow.response.headers.get('Location', '') - - entry = HAR.entries( - { - "startedDateTime": started_date_time, - "time": full_time, - "request": { - "method": flow.request.method, - "url": flow.request.url, - "httpVersion": request_http_version, - "cookies": request_cookies, - "headers": request_headers, - "queryString": request_query_string, - "headersSize": request_headers_size, - "bodySize": request_body_size, - }, - "response": { - "status": flow.response.status_code, - "statusText": flow.response.msg, - "httpVersion": response_http_version, - "cookies": response_cookies, - "headers": response_headers, - "content": { - "size": response_body_size, - "compression": response_body_compression, - "mimeType": response_mime_type}, - "redirectURL": response_redirect_url, - "headersSize": response_headers_size, - "bodySize": response_body_size, + + entry = HAR.entries({ + "startedDateTime": started_date_time, + "time": full_time, + "request": { + "method": flow.request.method, + "url": flow.request.url, + "httpVersion": flow.request.http_version, + "cookies": format_cookies(flow.request.cookies), + "headers": format_headers(flow.request.headers), + "queryString": request_query_string, + "headersSize": len(str(flow.request.headers)), + "bodySize": len(flow.request.content), + }, + "response": { + "status": flow.response.status_code, + "statusText": flow.response.msg, + "httpVersion": flow.response.http_version, + "cookies": format_cookies(flow.response.cookies), + "headers": format_headers(flow.response.headers), + "content": { + "size": response_body_size, + "compression": response_body_compression, + "mimeType": flow.response.headers.get('Content-Type', '') }, - "cache": {}, - "timings": timings, - }) - - # If the current url is in the page list of context.HARLog or does not have - # a referrer we add it as a new pages object. - if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get( - 'Referer', - None) is None: + "redirectURL": flow.response.headers.get('Location', ''), + "headersSize": len(str(flow.response.headers)), + "bodySize": response_body_size, + }, + "cache": {}, + "timings": timings, + }) + + # If the current url is in the page list of context.HARLog or + # does not have a referrer, we add it as a new pages object. + if (flow.request.url in context.HARLog.get_page_list() or + flow.request.headers.get('Referer') is None): page_id = context.HARLog.create_page_id() context.HARLog.add( HAR.pages({ @@ -215,7 +192,7 @@ def done(context): """ Called once on script shutdown, after any other events. """ - from pprint import pprint + import pprint import json json_dump = context.HARLog.json() @@ -239,6 +216,18 @@ def done(context): ) +def format_cookies(obj): + if obj: + return [{"name": k.strip(), "value": v[0]} for k, v in obj.items()] + return "" + + +def format_headers(obj): + if obj: + return [{"name": k, "value": v} for k, v in obj.fields] + return "" + + def print_attributes(obj, filter_string=None, hide_privates=False): """ Useful helper method to quickly get all attributes of an object and its diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index 3e9fa011..b1b860f8 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import os import re +import base64 import configargparse @@ -117,6 +118,15 @@ def parse_server_spec(url): return config.ServerSpec(scheme, address) +def parse_upstream_auth(auth): + pattern = re.compile(".+:") + if pattern.search(auth) is None: + raise configargparse.ArgumentTypeError( + "Invalid upstream auth specification: %s" % auth + ) + return "Basic" + " " + base64.b64encode(auth) + + def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -370,6 +380,15 @@ def proxy_options(parser): If your OpenSSL version supports ALPN, HTTP/2 is enabled by default. """ ) + parser.add_argument( + "--upstream-auth", + action="store", dest="upstream_auth", default=None, + type=parse_upstream_auth, + help=""" + Proxy Authentication: + username:password + """ + ) rawtcp = group.add_mutually_exclusive_group() rawtcp.add_argument("--raw-tcp", action="store_true", dest="rawtcp") rawtcp.add_argument("--no-raw-tcp", action="store_false", dest="rawtcp", diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index 52145516..6333de57 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -1,7 +1,10 @@ +import json import urllib -import netlib.http from textwrap import dedent +import netlib.http +from netlib.utils import parse_content_type + def curl_command(flow): data = "curl " @@ -53,8 +56,16 @@ def python_code(flow): data = "" if flow.request.body: - data = "\ndata = '''%s'''\n" % flow.request.body - args += "\n data=data," + json_obj = is_json(flow.request.headers, flow.request.body) + if json_obj: + # Without the separators field json.dumps() produces + # trailing white spaces: https://bugs.python.org/issue16333 + data = json.dumps(json_obj, indent=4, separators=(',', ': ')) + data = "\njson = %s\n" % data + args += "\n json=json," + else: + data = "\ndata = '''%s'''\n" % flow.request.body + args += "\n data=data," code = code.format( url=url, @@ -71,3 +82,14 @@ def python_code(flow): def raw_request(flow): data = netlib.http.http1.assemble_request(flow.request) return data + + +def is_json(headers, content): + if headers: + ct = parse_content_type(headers.get("content-type", "")) + if ct and "%s/%s" % (ct[0], ct[1]) == "application/json": + try: + return json.loads(content) + except ValueError: + return False + return False diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 394fe51a..0338945b 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -192,6 +192,9 @@ class HTTPRequest(MessageMixin, Request): def __hash__(self): return id(self) + def set_auth(self, auth): + self.data.headers.set_all("Proxy-Authorization", (auth,)) + def replace(self, pattern, repl, *args, **kwargs): """ Replaces a regular expression pattern with repl in the headers, the diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index 13d7903b..81e59fbb 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -179,6 +179,9 @@ class HttpLayer(Layer): try: flow = HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request + # set upstream auth + if self.mode == "upstream" and self.config.upstream_auth is not None: + flow.request.set_auth(self.config.upstream_auth) self.process_request_hook(flow) if not flow.response: diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 490cf20c..149d4710 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -53,6 +53,7 @@ class ProxyConfig: body_size_limit=None, mode="regular", upstream_server=None, + upstream_auth = None, authenticator=None, ignore_hosts=tuple(), tcp_hosts=tuple(), @@ -77,8 +78,10 @@ class ProxyConfig: self.mode = mode if upstream_server: self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) + self.upstream_auth = upstream_auth else: self.upstream_server = None + self.upstream_auth = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -110,7 +113,7 @@ def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server = "regular", None + mode, upstream_server, upstream_auth = "regular", None, None if options.transparent_proxy: c += 1 if not platform.resolver: @@ -127,6 +130,7 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy + upstream_auth = options.upstream_auth if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " @@ -189,6 +193,7 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, + upstream_auth=upstream_auth, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, http2=options.http2, diff --git a/test/mitmproxy/data/har_extractor.har b/test/mitmproxy/data/har_extractor.har new file mode 100644 index 00000000..2f5099b3 --- /dev/null +++ b/test/mitmproxy/data/har_extractor.har @@ -0,0 +1,78 @@ +{ + "test_response": { + "log": { + "__page_count__": 1, + "version": "1.2", + "creator": { + "comment": "", + "version": "0.1", + "name": "MITMPROXY HARExtractor" + }, + "pages": [ + { + "startedDateTime": "1993-08-24T14:41:12", + "id": "autopage_1", + "title": "http://address:22/path" + } + ], + "entries": [ + { + "pageref": "autopage_1", + "startedDateTime": "1993-08-24T14:41:12", + "cache": {}, + "request": { + "cookies": [], + "url": "http://address:22/path", + "queryString": [], + "headers": [ + { + "name": "header", + "value": "qvalue" + }, + { + "name": "content-length", + "value": "7" + } + ], + "headersSize": 35, + "httpVersion": "HTTP/1.1", + "method": "GET", + "bodySize": 7 + }, + "timings": { + "receive": 0, + "ssl": 1000, + "connect": 1000, + "send": 0, + "wait": 0 + }, + "time": 2000, + "response": { + "status": 200, + "cookies": [], + "statusText": "OK", + "content": { + "mimeType": "", + "compression": 0, + "size": 7 + }, + "headers": [ + { + "name": "content-length", + "value": "7" + }, + { + "name": "header-response", + "value": "svalue" + } + ], + "headersSize": 44, + "redirectURL": "", + "httpVersion": "HTTP/1.1", + "bodySize": 7 + } + } + ] + } + } +}
\ No newline at end of file diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/test_cmdline.py index 5a70f3e0..e75b7baf 100644 --- a/test/mitmproxy/test_cmdline.py +++ b/test/mitmproxy/test_cmdline.py @@ -1,4 +1,5 @@ import argparse +import base64 from mitmproxy import cmdline from . import tutils @@ -53,6 +54,16 @@ def test_parse_server_spec(): "http://") +def test_parse_upstream_auth(): + tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, "") + tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":") + tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":test") + assert cmdline.parse_upstream_auth( + "test:test") == "Basic" + " " + base64.b64encode("test:test") + assert cmdline.parse_upstream_auth( + "test:") == "Basic" + " " + base64.b64encode("test:") + + def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 2dce3fd6..3dc07427 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -1,6 +1,8 @@ +import json from textwrap import dedent import netlib.tutils +from netlib.http import Headers from mitmproxy import flow_export from . import tutils @@ -81,6 +83,35 @@ class TestExportPythonCode(): """).strip() assert flow_export.python_code(flow) == result + def test_post_json(self): + req_post.content = '{"name": "example", "email": "example@example.com"}' + req_post.headers = Headers(content_type="application/json") + flow = tutils.tflow(req=req_post) + result = dedent(""" + import requests + + url = 'http://address/path' + + headers = { + 'content-type': 'application/json', + } + + json = { + "name": "example", + "email": "example@example.com" + } + + response = requests.request( + method='POST', + url=url, + headers=headers, + json=json, + ) + + print(response.text) + """).strip() + assert flow_export.python_code(flow) == result + def test_patch(self): flow = tutils.tflow(req=req_patch) result = dedent(""" diff --git a/test/mitmproxy/test_har_extractor.py b/test/mitmproxy/test_har_extractor.py new file mode 100644 index 00000000..7838f713 --- /dev/null +++ b/test/mitmproxy/test_har_extractor.py @@ -0,0 +1,37 @@ +import json +import netlib.tutils +from . import tutils + +from examples import har_extractor + + +class Context(object): + pass + + +trequest = netlib.tutils.treq( + timestamp_start=746203272, + timestamp_end=746203272, +) + +tresponse = netlib.tutils.tresp( + timestamp_start=746203272, + timestamp_end=746203272, +) + + +def test_start(): + tutils.raises(ValueError, har_extractor.start, Context(), []) + + +def test_response(): + ctx = Context() + ctx.HARLog = har_extractor._HARLog([]) + ctx.seen_server = set() + + fl = tutils.tflow(req=trequest, resp=tresponse) + har_extractor.response(ctx, fl) + + with open(tutils.test_data.path("data/har_extractor.har")) as fp: + test_data = json.load(fp) + assert json.loads(ctx.HARLog.json()) == test_data["test_response"] diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 34b75b62..fddb851e 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -92,6 +92,10 @@ class TestProcessProxyOptions: self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") + self.assert_noerr("--upstream-auth", "test:test") + self.assert_err("expected one argument", "--upstream-auth") + self.assert_err("Invalid upstream auth specification", "--upstream-auth", "test") + self.assert_err("not allowed with", "-R", "http://localhost", "-T") def test_socks_auth(self): diff --git a/web/package.json b/web/package.json index a1b42d01..63a664ae 100644 --- a/web/package.json +++ b/web/package.json @@ -5,6 +5,7 @@ "test": "jest ./src/js" }, "jest": { + "scriptPreprocessor": "<rootDir>/node_modules/babel-jest", "testPathDirs": [ "./src/js" ], @@ -25,6 +26,7 @@ }, "devDependencies": { "babel-core": "^6.5.2", + "babel-jest": "^6.0.1", "babel-preset-es2015": "^6.5.0", "babel-preset-react": "^6.5.0", "babelify": "^7.2.0", @@ -41,6 +43,7 @@ "gulp-rename": "^1.2.2", "gulp-sourcemaps": "^1.6.0", "gulp-util": "^3.0.7", + "jest": "^0.1.40", "lodash": "^4.5.1", "uglifyify": "^3.0.1", "vinyl-buffer": "^1.0.0", diff --git a/web/src/js/tests/utils.js b/web/src/js/tests/utils.js index 1b6de264..acbadc92 100644 --- a/web/src/js/tests/utils.js +++ b/web/src/js/tests/utils.js @@ -1,8 +1,9 @@ jest.dontMock("jquery"); jest.dontMock("../utils"); +import {formatSize} from "../utils.js" + describe("utils", function () { - import {formatSize} from "../utils.js" it("formatSize", function(){ expect(formatSize(1024)).toEqual("1kb"); expect(formatSize(0)).toEqual("0"); |