From a44062effb5dbfb659943af6a3da5ae69f02efbc Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Wed, 23 Mar 2016 01:49:18 -0700 Subject: Flow export to locust.io load test tool. --- mitmproxy/flow_export.py | 123 +++++++++++++++++++++++- test/mitmproxy/test_flow_export.py | 189 ++++++++++++++++++++++++++++++++++++- 2 files changed, 308 insertions(+), 4 deletions(-) diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index 6333de57..a1e07953 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -1,10 +1,11 @@ import json -import urllib from textwrap import dedent import netlib.http from netlib.utils import parse_content_type +import re +from six.moves.urllib.parse import urlparse, quote, quote_plus def curl_command(flow): data = "curl " @@ -38,7 +39,7 @@ def python_code(flow): print(response.text) """).strip() - components = map(lambda x: urllib.quote(x, safe=""), flow.request.path_components) + components = map(lambda x: quote(x, safe=""), flow.request.path_components) url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) args = "" @@ -93,3 +94,121 @@ def is_json(headers, content): except ValueError: return False return False + + +def locust_code(flow): + code = dedent(""" + from locust import HttpLocust, TaskSet, task + + class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.flow() + + @task() + def flow(self): + url = '{url}' + {headers}{params}{data} + self.response = self.client.request( + method='{method}', + url=url,{args} + ) + + class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait=1000 + max_wait=3000 + + """).strip() + + components = map(lambda x: quote(x, safe=""), flow.request.path_components) + url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) + + args = "" + headers = "" + if flow.request.headers: + lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]] + headers += "\n headers = {\n%s }\n" % "".join(lines) + args += "\n headers=headers," + + params = "" + if flow.request.query: + lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.query] + params = "\n params = {\n%s }\n" % "".join(lines) + args += "\n params=params," + + data = "" + if flow.request.body: + data = "\n data = '''%s'''\n" % flow.request.body + args += "\n data=data," + + code = code.format( + url=url, + headers=headers, + params=params, + data=data, + method=flow.request.method, + args=args, + ) + + host = flow.request.scheme + "://" + flow.request.host + code = code.replace(host, "' + self.locust.host +'") + code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) +'") + code = code.replace(quote(host), "' + quote(self.locust.host) +'") + + return code + + +def locust_task(flow): + code = dedent(""" + @task() + def {name}(self): + url = '{url}' + {headers}{params}{data} + self.response = self.client.request( + method='{method}', + url=url,{args} + ) + """).strip() + + components = map(lambda x: quote(x, safe=""), flow.request.path_components) + file_name = "_".join(components) + name = re.sub('\W|^(?=\d)','_', file_name) + url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) + + args = "" + headers = "" + if flow.request.headers: + lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]] + headers += "\n headers = {\n%s }\n" % "".join(lines) + args += "\n headers=headers," + + params = "" + if flow.request.query: + lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.query] + params = "\n params = {\n%s }\n" % "".join(lines) + args += "\n params=params," + + data = "" + if flow.request.body: + data = "\n data = '''%s'''\n" % flow.request.body + args += "\n data=data," + + code = code.format( + name=name, + url=url, + headers=headers, + params=params, + data=data, + method=flow.request.method, + args=args, + ) + + host = flow.request.scheme + "://" + flow.request.host + code = code.replace(host, "' + self.locust.host +'") + code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) +'") + code = code.replace(quote(host), "' + quote(self.locust.host) +'") + + code = "\n".join(" " + i for i in code.splitlines()) + + return code diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 62161d5d..a4264e10 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -142,7 +142,7 @@ class TestExportPythonCode(): assert flow_export.python_code(flow) == result -def TestRawRequest(): +class TestRawRequest(): def test_get(self): flow = tutils.tflow(req=req_get) @@ -159,9 +159,10 @@ def TestRawRequest(): flow = tutils.tflow(req=req_post) result = dedent(""" POST /path HTTP/1.1\r + content-type: application/json\r host: address:22\r \r - content + {"name": "example", "email": "example@example.com"} """).strip() assert flow_export.raw_request(flow) == result @@ -176,3 +177,187 @@ def TestRawRequest(): content """).strip() assert flow_export.raw_request(flow) == result + +class TestExportLocustCode(): + + def test_get(self): + flow = tutils.tflow(req=req_get) + result = """ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.flow() + + @task() + def flow(self): + url = '' + self.locust.host +'/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + self.response = self.client.request( + method='GET', + url=url, + headers=headers, + ) + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait=1000 + max_wait=3000 + """.strip() + + assert flow_export.locust_code(flow) == result + + def test_post(self): + req_post.content = '''content''' + req_post.headers = '' + flow = tutils.tflow(req=req_post) + result = """ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.flow() + + @task() + def flow(self): + url = '' + self.locust.host +'/path' + + data = '''content''' + + self.response = self.client.request( + method='POST', + url=url, + data=data, + ) + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait=1000 + max_wait=3000 + + """.strip() + + assert flow_export.locust_code(flow) == result + + + def test_patch(self): + flow = tutils.tflow(req=req_patch) + result = """ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.flow() + + @task() + def flow(self): + url = '' + self.locust.host +'/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + params = { + 'query': 'param', + } + + data = '''content''' + + self.response = self.client.request( + method='PATCH', + url=url, + headers=headers, + params=params, + data=data, + ) + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait=1000 + max_wait=3000 + + """.strip() + + assert flow_export.locust_code(flow) == result + + +class TestExportLocustTask(): + + def test_get(self): + flow = tutils.tflow(req=req_get) + result = ' ' + """ + @task() + def path(self): + url = '' + self.locust.host +'/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + self.response = self.client.request( + method='GET', + url=url, + headers=headers, + ) + """.strip() + + assert flow_export.locust_task(flow) == result + + def test_post(self): + flow = tutils.tflow(req=req_post) + result = ' ' + """ + @task() + def path(self): + url = '' + self.locust.host +'/path' + + data = '''content''' + + self.response = self.client.request( + method='POST', + url=url, + data=data, + ) + + """.strip() + + assert flow_export.locust_task(flow) == result + + + def test_patch(self): + flow = tutils.tflow(req=req_patch) + result = ' ' + """ + @task() + def path(self): + url = '' + self.locust.host +'/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + params = { + 'query': 'param', + } + + data = '''content''' + + self.response = self.client.request( + method='PATCH', + url=url, + headers=headers, + params=params, + data=data, + ) + """.strip() + + assert flow_export.locust_task(flow) == result -- cgit v1.2.3 From 5b07e8b3af856ced512702b770a40030cdb200a2 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Thu, 24 Mar 2016 20:29:53 -0700 Subject: Add UI shortcuts. --- mitmproxy/console/common.py | 2 ++ mitmproxy/console/flowlist.py | 2 ++ mitmproxy/console/flowview.py | 2 ++ mitmproxy/flow_export.py | 3 +++ test/mitmproxy/test_flow_export.py | 9 +++++++++ 5 files changed, 18 insertions(+) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index c29ffddc..27e6b6db 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -286,6 +286,8 @@ def export_prompt(k, flow): "c": flow_export.curl_command, "p": flow_export.python_code, "r": flow_export.raw_request, + "l": flow_export.locust_code, + "t": flow_export.locust_task, } if k in exporters: copy_to_clipboard_or_prompt(exporters[k](flow)) diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index c2201055..78b30231 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -265,6 +265,8 @@ class ConnectionItem(urwid.WidgetWrap): ("as curl command", "c"), ("as python code", "p"), ("as raw request", "r"), + ("as locust code", "l"), + ("as locust task", "t"), ), callback = common.export_prompt, args = (self.flow,) diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index f74ab140..ba2aad6a 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -585,6 +585,8 @@ class FlowView(tabs.Tabs): ("as curl command", "c"), ("as python code", "p"), ("as raw request", "r"), + ("as locust code", "l"), + ("as locust task", "t"), ), callback = common.export_prompt, args = (self.flow,) diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index a1e07953..f91a59a1 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -114,6 +114,9 @@ def locust_code(flow): url=url,{args} ) + ### Additional tasks can go here ### + + class WebsiteUser(HttpLocust): task_set = UserBehavior min_wait=1000 diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index a4264e10..6654d990 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -205,6 +205,9 @@ class UserBehavior(TaskSet): headers=headers, ) + ### Additional tasks can go here ### + + class WebsiteUser(HttpLocust): task_set = UserBehavior min_wait=1000 @@ -237,6 +240,9 @@ class UserBehavior(TaskSet): data=data, ) + ### Additional tasks can go here ### + + class WebsiteUser(HttpLocust): task_set = UserBehavior min_wait=1000 @@ -280,6 +286,9 @@ class UserBehavior(TaskSet): data=data, ) + ### Additional tasks can go here ### + + class WebsiteUser(HttpLocust): task_set = UserBehavior min_wait=1000 -- cgit v1.2.3 From 9f77c80a327e7c409f0971b9d83ff3a67c2da231 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Fri, 25 Mar 2016 17:29:42 -0700 Subject: pep8 --- mitmproxy/flow_export.py | 16 ++++++++-------- test/mitmproxy/test_flow_export.py | 24 ++++++++++++------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index f91a59a1..a9fd5cde 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -119,8 +119,8 @@ def locust_code(flow): class WebsiteUser(HttpLocust): task_set = UserBehavior - min_wait=1000 - max_wait=3000 + min_wait = 1000 + max_wait = 3000 """).strip() @@ -155,9 +155,9 @@ def locust_code(flow): ) host = flow.request.scheme + "://" + flow.request.host - code = code.replace(host, "' + self.locust.host +'") - code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) +'") - code = code.replace(quote(host), "' + quote(self.locust.host) +'") + code = code.replace(host, "' + self.locust.host + '") + code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) + '") + code = code.replace(quote(host), "' + quote(self.locust.host) + '") return code @@ -208,9 +208,9 @@ def locust_task(flow): ) host = flow.request.scheme + "://" + flow.request.host - code = code.replace(host, "' + self.locust.host +'") - code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) +'") - code = code.replace(quote(host), "' + quote(self.locust.host) +'") + code = code.replace(host, "' + self.locust.host + '") + code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) + '") + code = code.replace(quote(host), "' + quote(self.locust.host) + '") code = "\n".join(" " + i for i in code.splitlines()) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 6654d990..6c9a6756 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -192,7 +192,7 @@ class UserBehavior(TaskSet): @task() def flow(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -210,8 +210,8 @@ class UserBehavior(TaskSet): class WebsiteUser(HttpLocust): task_set = UserBehavior - min_wait=1000 - max_wait=3000 + min_wait = 1000 + max_wait = 3000 """.strip() assert flow_export.locust_code(flow) == result @@ -230,7 +230,7 @@ class UserBehavior(TaskSet): @task() def flow(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' data = '''content''' @@ -245,8 +245,8 @@ class UserBehavior(TaskSet): class WebsiteUser(HttpLocust): task_set = UserBehavior - min_wait=1000 - max_wait=3000 + min_wait = 1000 + max_wait = 3000 """.strip() @@ -265,7 +265,7 @@ class UserBehavior(TaskSet): @task() def flow(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -291,8 +291,8 @@ class UserBehavior(TaskSet): class WebsiteUser(HttpLocust): task_set = UserBehavior - min_wait=1000 - max_wait=3000 + min_wait = 1000 + max_wait = 3000 """.strip() @@ -306,7 +306,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -327,7 +327,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' data = '''content''' @@ -347,7 +347,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host +'/path' + url = '' + self.locust.host + '/path' headers = { 'header': 'qvalue', -- cgit v1.2.3 From 2f285a6015abaf75ced34231f698537ef210ffed Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 26 Mar 2016 11:26:42 +0800 Subject: Setting CONTENT_MISSING to None --- netlib/http/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netlib/http/message.py b/netlib/http/message.py index e3d8ce37..4ddc1b81 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -7,7 +7,7 @@ import six from .headers import Headers from .. import encoding, utils -CONTENT_MISSING = 0 +CONTENT_MISSING = None if six.PY2: # pragma: nocover _native = lambda x: x -- cgit v1.2.3 From 08ff00f36d49ef53b1d20d7635a74fa46a249c17 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 26 Mar 2016 16:00:51 +0800 Subject: replace CONTENT_MISSING with None. --- mitmproxy/console/common.py | 7 +++---- mitmproxy/console/flowview.py | 6 +++--- mitmproxy/dump.py | 5 ++--- mitmproxy/flow.py | 4 ++-- mitmproxy/models/http.py | 10 ++++------ mitmproxy/protocol/http.py | 8 ++++---- mitmproxy/web/app.py | 3 +-- netlib/http/__init__.py | 4 ++-- netlib/http/http1/assemble.py | 10 ++++------ netlib/http/message.py | 2 -- test/mitmproxy/test_dump.py | 7 +++---- test/mitmproxy/test_flow.py | 6 +++--- test/mitmproxy/test_server.py | 6 +++--- test/netlib/http/http1/test_assemble.py | 6 +++--- 14 files changed, 37 insertions(+), 47 deletions(-) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index c29ffddc..746a67f1 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -4,7 +4,6 @@ import urwid import urwid.util import os -from netlib.http import CONTENT_MISSING import netlib.utils from .. import utils @@ -256,7 +255,7 @@ def copy_flow_format_data(part, scope, flow): else: data = "" if scope in ("q", "a"): - if flow.request.content is None or flow.request.content == CONTENT_MISSING: + if flow.request.content is None: return None, "Request content is missing" with decoded(flow.request): if part == "h": @@ -269,7 +268,7 @@ def copy_flow_format_data(part, scope, flow): # Add padding between request and response data += "\r\n" * 2 if scope in ("s", "a") and flow.response: - if flow.response.content is None or flow.response.content == CONTENT_MISSING: + if flow.response.content is None: return None, "Response content is missing" with decoded(flow.response): if part == "h": @@ -418,7 +417,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False): if f.response: if f.response.content: contentdesc = netlib.utils.pretty_size(len(f.response.content)) - elif f.response.content == CONTENT_MISSING: + elif f.response.content is None: contentdesc = "[content missing]" else: contentdesc = "[no content]" diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index f74ab140..3b86c920 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -7,7 +7,7 @@ import math import urwid from netlib import odict -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from . import common, grideditor, signals, searchable, tabs from . import flowdetailview from .. import utils, controller, contentviews @@ -169,7 +169,7 @@ class FlowView(tabs.Tabs): self.show() def content_view(self, viewmode, message): - if message.content == CONTENT_MISSING: + if message.content is None: msg, body = "", [urwid.Text([("error", "[content missing]")])] return msg, body else: @@ -510,7 +510,7 @@ class FlowView(tabs.Tabs): def delete_body(self, t): if t == "m": - val = CONTENT_MISSING + val = None else: val = None if self.tab_offset == TAB_REQ: diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 7b4609b4..631e4d2e 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -5,7 +5,6 @@ import click import itertools from netlib import tcp -from netlib.http import CONTENT_MISSING import netlib.utils from . import flow, filt, contentviews from .exceptions import ContentViewException @@ -180,7 +179,7 @@ class DumpMaster(flow.FlowMaster): ) self.echo(headers, indent=4) if self.o.flow_detail >= 3: - if message.content == CONTENT_MISSING: + if message.content is None: self.echo("(content missing)", indent=4) elif message.content: self.echo("") @@ -283,7 +282,7 @@ class DumpMaster(flow.FlowMaster): code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) reason = click.style(flow.response.reason, fg=code_color, bold=True) - if flow.response.content == CONTENT_MISSING: + if flow.response.content is None: size = "(content missing)" else: size = netlib.utils.pretty_size(len(flow.response.content)) diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index d656bc4d..fbf102b5 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -16,7 +16,7 @@ from six.moves import urllib from netlib import wsgi from netlib.exceptions import HttpException -from netlib.http import CONTENT_MISSING, Headers, http1 +from netlib.http import Headers, http1 from . import controller, tnetstring, filt, script, version, flow_format_compat from .onboarding import app from .proxy.config import HostMatcher @@ -942,7 +942,7 @@ class FlowMaster(controller.Master): return "Can't replay live request." if f.intercepted: return "Can't replay while intercepting..." - if f.request.content == CONTENT_MISSING: + if f.request.content is None: return "Can't replay request with missing content..." if f.request: f.backup() diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index a80e11b0..428b1ba6 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -84,9 +84,8 @@ class HTTPRequest(MessageMixin, Request): headers: Headers object - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. + content: Content of the request, the value is None if there is content + associated, but not present. form_in: The request form which mitmproxy has received. The following values are possible: @@ -226,9 +225,8 @@ class HTTPResponse(MessageMixin, Response): headers: Headers object - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. + content: Content of the request, the value is None if there is content + associated, but not present. timestamp_start: Timestamp indicating when request transmission started diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index a2745eac..7f134efe 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -6,7 +6,7 @@ import six from netlib import tcp from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException -from netlib.http import Headers, CONTENT_MISSING +from netlib.http import Headers from h2.exceptions import H2Error @@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer): yield "this is a generator" # pragma: no cover def send_response(self, response): - if response.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if response.content == None: + raise HttpException("Cannot assemble flow with None content") self.send_response_headers(response) self.send_response_body(response, [response.content]) @@ -318,7 +318,7 @@ class HttpLayer(Layer): raise Kill() if flow.response.stream: - flow.response.data.content = CONTENT_MISSING + flow.response.data.content = None else: flow.response.data.content = b"".join(self.read_response_body( flow.request, diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 2cac2ab9..8a8329e7 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -8,7 +8,6 @@ import logging import json import base64 -from netlib.http import CONTENT_MISSING from .. import version, filt @@ -26,7 +25,7 @@ def _strip_content(flow_state): continue if message["content"]: message["contentLength"] = len(message["content"]) - elif message["content"] == CONTENT_MISSING: + elif message["content"] is None: message["contentLength"] = None else: message["contentLength"] = 0 diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index fd632cd5..917080f7 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division from .request import Request from .response import Response from .headers import Headers -from .message import decoded, CONTENT_MISSING +from .message import decoded from . import http1, http2 __all__ = [ "Request", "Response", "Headers", - "decoded", "CONTENT_MISSING", + "decoded", "http1", "http2", ] diff --git a/netlib/http/http1/assemble.py b/netlib/http/http1/assemble.py index 785ee8d3..db5a49ce 100644 --- a/netlib/http/http1/assemble.py +++ b/netlib/http/http1/assemble.py @@ -3,12 +3,10 @@ from __future__ import absolute_import, print_function, division from ... import utils import itertools from ...exceptions import HttpException -from .. import CONTENT_MISSING - def assemble_request(request): - if request.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if request.content == None: + raise HttpException("Cannot assemble flow with None content") head = assemble_request_head(request) body = b"".join(assemble_body(request.data.headers, [request.data.content])) return head + body @@ -21,8 +19,8 @@ def assemble_request_head(request): def assemble_response(response): - if response.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if response.content == None: + raise HttpException("Cannot assemble flow with None content") head = assemble_response_head(response) body = b"".join(assemble_body(response.data.headers, [response.data.content])) return head + body diff --git a/netlib/http/message.py b/netlib/http/message.py index 4ddc1b81..1df0f087 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -7,8 +7,6 @@ import six from .headers import Headers from .. import encoding, utils -CONTENT_MISSING = None - if six.PY2: # pragma: nocover _native = lambda x: x _always_bytes = lambda x: x diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 2228a732..7e772881 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -4,7 +4,6 @@ from mitmproxy.exceptions import ContentViewException from mitmproxy.models import HTTPResponse import netlib.tutils -from netlib.http import CONTENT_MISSING from mitmproxy import dump, flow from mitmproxy.proxy import Log @@ -38,7 +37,7 @@ def test_strfuncs(): flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=CONTENT_MISSING) + flow.response = netlib.tutils.tresp(content=None) flow.response.is_replay = True flow.response.status_code = 300 m.echo_flow(flow) @@ -104,10 +103,10 @@ class TestDumpMaster: o = dump.Options(flow_detail=3) m = dump.DumpMaster(None, o, outfile=cs) f = tutils.tflow() - f.request.content = CONTENT_MISSING + f.request.content = None m.handle_request(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp()) - f.response.content = CONTENT_MISSING + f.response.content = None m.handle_response(f) assert "content missing" in cs.getvalue() diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 33d2b8f9..2353935b 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -8,7 +8,7 @@ import mock import netlib.utils from netlib import odict -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from mitmproxy import filt, controller, tnetstring, flow from mitmproxy.models import Error from mitmproxy.models import Flow @@ -465,7 +465,7 @@ class TestFlow(object): def test_replace_no_content(self): f = tutils.tflow() - f.request.content = CONTENT_MISSING + f.request.content = None assert f.replace("foo", "bar") == 0 def test_replace(self): @@ -751,7 +751,7 @@ class TestFlowMaster: s = flow.State() fm = flow.FlowMaster(None, s) f = tutils.tflow(resp=True) - f.request.content = CONTENT_MISSING + f.request.content = None assert "missing" in fm.replay_request(f) f.intercepted = True diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 26e53e8a..dc72f032 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -8,7 +8,7 @@ from netlib.tcp import Address import netlib.tutils from netlib import tcp, http, socks from netlib.certutils import SSLCert -from netlib.http import authentication, CONTENT_MISSING, http1 +from netlib.http import authentication, http1 from netlib.tutils import raises from pathod import pathoc, pathod @@ -281,7 +281,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.pathod("200:b@3k") assert self.master.state.view[-1].response.stream - assert self.master.state.view[-1].response.content == CONTENT_MISSING + assert self.master.state.view[-1].response.content is None self.master.set_stream_large_bodies(None) def test_stream_modify(self): @@ -816,7 +816,7 @@ class MasterIncomplete(tservers.TestMaster): def handle_request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) - resp.content = CONTENT_MISSING + resp.content = None f.reply(resp) diff --git a/test/netlib/http/http1/test_assemble.py b/test/netlib/http/http1/test_assemble.py index 31a62438..8dcbae8e 100644 --- a/test/netlib/http/http1/test_assemble.py +++ b/test/netlib/http/http1/test_assemble.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, print_function, division from netlib.exceptions import HttpException -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from netlib.http.http1.assemble import ( assemble_request, assemble_request_head, assemble_response, assemble_response_head, _assemble_request_line, _assemble_request_headers, @@ -20,7 +20,7 @@ def test_assemble_request(): ) with raises(HttpException): - assemble_request(treq(content=CONTENT_MISSING)) + assemble_request(treq(content=None)) def test_assemble_request_head(): @@ -41,7 +41,7 @@ def test_assemble_response(): ) with raises(HttpException): - assemble_response(tresp(content=CONTENT_MISSING)) + assemble_response(tresp(content=None)) def test_assemble_response_head(): -- cgit v1.2.3 From 53e15f778df316a223847cb81e4f2eb25d29a539 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 26 Mar 2016 16:17:55 +0800 Subject: update document for the removal of CONTENT_MISSING --- docs/features/responsestreaming.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 8975c1f8..9dc27bf4 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -48,8 +48,7 @@ Implementation Details ---------------------- When response streaming is enabled, portions of the code which would have otherwise performed -changes on the response body will see an empty response body instead -(:py:data:`netlib.http.CONTENT_MISSING`). Any modifications will be ignored. +changes on the response body will see an empty response body. Any modifications will be ignored. Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a ``Transfer-Encoding: chunked`` header, the response will be streamed one chunk at a time. -- cgit v1.2.3 From 66bd27e6f9018832f99be1c211e76c8f71165c92 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 26 Mar 2016 17:49:22 +0800 Subject: update comments --- mitmproxy/models/http.py | 2 +- mitmproxy/protocol/http.py | 4 ++-- netlib/http/http1/assemble.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 428b1ba6..40460182 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -225,7 +225,7 @@ class HTTPResponse(MessageMixin, Response): headers: Headers object - content: Content of the request, the value is None if there is content + content: Content of the response, the value is None if there is content associated, but not present. timestamp_start: Timestamp indicating when request transmission started diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index 7f134efe..22e71422 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer): yield "this is a generator" # pragma: no cover def send_response(self, response): - if response.content == None: - raise HttpException("Cannot assemble flow with None content") + if response.content is None: + raise HttpException("Cannot assemble flow with missing content") self.send_response_headers(response) self.send_response_body(response, [response.content]) diff --git a/netlib/http/http1/assemble.py b/netlib/http/http1/assemble.py index db5a49ce..f06ad5a1 100644 --- a/netlib/http/http1/assemble.py +++ b/netlib/http/http1/assemble.py @@ -5,8 +5,8 @@ import itertools from ...exceptions import HttpException def assemble_request(request): - if request.content == None: - raise HttpException("Cannot assemble flow with None content") + if request.content is None: + raise HttpException("Cannot assemble flow with missing content") head = assemble_request_head(request) body = b"".join(assemble_body(request.data.headers, [request.data.content])) return head + body @@ -19,8 +19,8 @@ def assemble_request_head(request): def assemble_response(response): - if response.content == None: - raise HttpException("Cannot assemble flow with None content") + if response.content is None: + raise HttpException("Cannot assemble flow with missing content") head = assemble_response_head(response) body = b"".join(assemble_body(response.data.headers, [response.data.content])) return head + body -- cgit v1.2.3 From 6e4af64050fe9a0efbfa73193131065b96feca3c Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sun, 27 Mar 2016 09:16:40 +0800 Subject: minor fix about if-else statement --- mitmproxy/web/app.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 8a8329e7..cf3c8bdd 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -25,10 +25,8 @@ def _strip_content(flow_state): continue if message["content"]: message["contentLength"] = len(message["content"]) - elif message["content"] is None: - message["contentLength"] = None else: - message["contentLength"] = 0 + message["contentLength"] = None del message["content"] if "backup" in flow_state: -- cgit v1.2.3 From ab7e80085af891c494bc274c0cde27d49f89f254 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 27 Mar 2016 11:59:27 +0200 Subject: increase test timeout --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 96b23026..b1d62172 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,7 @@ ignore = E251 [pytest] testpaths = test -addopts = --timeout 30 -s +addopts = --timeout 60 --capture=no [coverage:run] branch = True -- cgit v1.2.3 From ec68d8b8e4a9fc24e45379359f96b3ebc30e381a Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 27 Mar 2016 12:02:41 +0200 Subject: s/nocover/no cover/g according to coveralls docs --- mitmproxy/script/script.py | 2 +- netlib/http/authentication.py | 2 +- netlib/http/headers.py | 6 +++--- netlib/http/message.py | 6 +++--- netlib/http/request.py | 28 ++++++++++++++-------------- netlib/http/response.py | 8 ++++---- pathod/pathoc.py | 4 ++-- pathod/pathoc_cmdline.py | 2 +- pathod/pathod.py | 2 +- pathod/pathod_cmdline.py | 2 +- pathod/utils.py | 2 +- setup.cfg | 1 - 12 files changed, 32 insertions(+), 33 deletions(-) diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py index f142daca..5a8334c4 100644 --- a/mitmproxy/script/script.py +++ b/mitmproxy/script/script.py @@ -47,7 +47,7 @@ class Script(object): if os.name == "nt": # pragma: no cover backslashes = shlex.split(command, posix=False)[0].count("\\") command = command.replace("\\", "\\\\", backslashes) - args = shlex.split(command) # pragma: nocover + args = shlex.split(command) # pragma: no cover args[0] = os.path.expanduser(args[0]) if not os.path.exists(args[0]): raise ScriptException( diff --git a/netlib/http/authentication.py b/netlib/http/authentication.py index d769abe5..6db70fdd 100644 --- a/netlib/http/authentication.py +++ b/netlib/http/authentication.py @@ -140,7 +140,7 @@ class AuthAction(Action): authenticator = BasicProxyAuth(passman, "mitmproxy") setattr(namespace, self.dest, authenticator) - def getPasswordManager(self, s): # pragma: nocover + def getPasswordManager(self, s): # pragma: no cover raise NotImplementedError() diff --git a/netlib/http/headers.py b/netlib/http/headers.py index 78404796..9b8fdae4 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, print_function, division import copy try: from collections.abc import MutableMapping -except ImportError: # pragma: nocover +except ImportError: # pragma: no cover from collections import MutableMapping # Workaround for Python < 3.3 @@ -16,7 +16,7 @@ import six from netlib.utils import always_byte_args, always_bytes, Serializable -if six.PY2: # pragma: nocover +if six.PY2: # pragma: no cover _native = lambda x: x _always_bytes = lambda x: x _always_byte_args = lambda x: x @@ -106,7 +106,7 @@ class Headers(MutableMapping, Serializable): else: return b"" - if six.PY2: # pragma: nocover + if six.PY2: # pragma: no cover __str__ = __bytes__ @_always_byte_args diff --git a/netlib/http/message.py b/netlib/http/message.py index 1df0f087..b6d846ba 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -7,7 +7,7 @@ import six from .headers import Headers from .. import encoding, utils -if six.PY2: # pragma: nocover +if six.PY2: # pragma: no cover _native = lambda x: x _always_bytes = lambda x: x else: @@ -180,12 +180,12 @@ class Message(utils.Serializable): # Legacy @property - def body(self): # pragma: nocover + def body(self): # pragma: no cover warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning) return self.content @body.setter - def body(self, body): # pragma: nocover + def body(self, body): # pragma: no cover warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning) self.content = body diff --git a/netlib/http/request.py b/netlib/http/request.py index 1a9b6f18..d35c1874 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -100,7 +100,7 @@ class Request(Message): Setting the host attribute also updates the host header, if present. """ - if six.PY2: # pragma: nocover + if six.PY2: # pragma: no cover return self.data.host if not self.data.host: @@ -324,59 +324,59 @@ class Request(Message): # Legacy - def get_cookies(self): # pragma: nocover + def get_cookies(self): # pragma: no cover warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) return self.cookies - def set_cookies(self, odict): # pragma: nocover + def set_cookies(self, odict): # pragma: no cover warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) self.cookies = odict - def get_query(self): # pragma: nocover + def get_query(self): # pragma: no cover warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning) return self.query or ODict([]) - def set_query(self, odict): # pragma: nocover + def set_query(self, odict): # pragma: no cover warnings.warn(".set_query is deprecated, use .query instead.", DeprecationWarning) self.query = odict - def get_path_components(self): # pragma: nocover + def get_path_components(self): # pragma: no cover warnings.warn(".get_path_components is deprecated, use .path_components instead.", DeprecationWarning) return self.path_components - def set_path_components(self, lst): # pragma: nocover + def set_path_components(self, lst): # pragma: no cover warnings.warn(".set_path_components is deprecated, use .path_components instead.", DeprecationWarning) self.path_components = lst - def get_form_urlencoded(self): # pragma: nocover + def get_form_urlencoded(self): # pragma: no cover warnings.warn(".get_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) return self.urlencoded_form or ODict([]) - def set_form_urlencoded(self, odict): # pragma: nocover + def set_form_urlencoded(self, odict): # pragma: no cover warnings.warn(".set_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) self.urlencoded_form = odict - def get_form_multipart(self): # pragma: nocover + def get_form_multipart(self): # pragma: no cover warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning) return self.multipart_form or ODict([]) @property - def form_in(self): # pragma: nocover + def form_in(self): # pragma: no cover warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning) return self.first_line_format @form_in.setter - def form_in(self, form_in): # pragma: nocover + def form_in(self, form_in): # pragma: no cover warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning) self.first_line_format = form_in @property - def form_out(self): # pragma: nocover + def form_out(self): # pragma: no cover warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning) return self.first_line_format @form_out.setter - def form_out(self, form_out): # pragma: nocover + def form_out(self, form_out): # pragma: no cover warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning) self.first_line_format = form_out diff --git a/netlib/http/response.py b/netlib/http/response.py index 8f4d6215..da2c8142 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -97,20 +97,20 @@ class Response(Message): # Legacy - def get_cookies(self): # pragma: nocover + def get_cookies(self): # pragma: no cover warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) return self.cookies - def set_cookies(self, odict): # pragma: nocover + def set_cookies(self, odict): # pragma: no cover warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) self.cookies = odict @property - def msg(self): # pragma: nocover + def msg(self): # pragma: no cover warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning) return self.reason @msg.setter - def msg(self, reason): # pragma: nocover + def msg(self, reason): # pragma: no cover warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning) self.reason = reason diff --git a/pathod/pathoc.py b/pathod/pathoc.py index 86661f98..0e6d3ca7 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -208,7 +208,7 @@ class Pathoc(tcp.TCPClient): self.ws_framereader = None if self.use_http2: - if not tcp.HAS_ALPN: # pragma: nocover + if not tcp.HAS_ALPN: # pragma: no cover log.write_raw( self.fp, "HTTP/2 requires ALPN support. " @@ -458,7 +458,7 @@ class Pathoc(tcp.TCPClient): # TODO: do something -def main(args): # pragma: nocover +def main(args): # pragma: no cover memo = set([]) trycount = 0 p = None diff --git a/pathod/pathoc_cmdline.py b/pathod/pathoc_cmdline.py index bf827a9a..b59704f3 100644 --- a/pathod/pathoc_cmdline.py +++ b/pathod/pathoc_cmdline.py @@ -221,6 +221,6 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): return args -def go_pathoc(): # pragma: nocover +def go_pathoc(): # pragma: no cover args = args_pathoc(sys.argv) pathoc.main(args) diff --git a/pathod/pathod.py b/pathod/pathod.py index 55e75074..b80da887 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -430,7 +430,7 @@ class Pathod(tcp.TCPServer): return self.log -def main(args): # pragma: nocover +def main(args): # pragma: no cover ssloptions = SSLOptions( cn=args.cn, confdir=args.confdir, diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py index c9272249..1f972a49 100644 --- a/pathod/pathod_cmdline.py +++ b/pathod/pathod_cmdline.py @@ -226,6 +226,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): return args -def go_pathod(): # pragma: nocover +def go_pathod(): # pragma: no cover args = args_pathod(sys.argv) pathod.main(args) diff --git a/pathod/utils.py b/pathod/utils.py index a1109a3c..1e5bd9a4 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -98,7 +98,7 @@ class Data(object): data = Data(__name__) -def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: nocover +def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover try: pid = os.fork() if pid > 0: diff --git a/setup.cfg b/setup.cfg index b1d62172..7849ca07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,5 @@ omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py [coverage:report] show_missing = True exclude_lines = - pragma: nocover pragma: no cover raise NotImplementedError() -- cgit v1.2.3 From 04cb099b15b281cb4ff76be0e5d7ec6efb199bbf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 27 Mar 2016 13:04:19 +0200 Subject: improve flow export tests --- test/mitmproxy/test_flow_export.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 62161d5d..d98759bb 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -1,3 +1,4 @@ +import json from textwrap import dedent import netlib.tutils @@ -142,7 +143,7 @@ class TestExportPythonCode(): assert flow_export.python_code(flow) == result -def TestRawRequest(): +class TestRawRequest(): def test_get(self): flow = tutils.tflow(req=req_get) @@ -159,9 +160,10 @@ def TestRawRequest(): flow = tutils.tflow(req=req_post) result = dedent(""" POST /path HTTP/1.1\r + content-type: application/json\r host: address:22\r \r - content + {"name": "example", "email": "example@example.com"} """).strip() assert flow_export.raw_request(flow) == result @@ -176,3 +178,22 @@ def TestRawRequest(): content """).strip() assert flow_export.raw_request(flow) == result + +class TestIsJson(): + + def test_empty(self): + assert flow_export.is_json(None, None) == False + + def test_json_type(self): + headers = Headers(content_type="application/json") + assert flow_export.is_json(headers, "foobar") == False + + def test_valid(self): + headers = Headers(content_type="application/foobar") + j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + assert j == False + + def test_valid(self): + headers = Headers(content_type="application/json") + j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + assert isinstance(j, dict) -- cgit v1.2.3 From dfcfa6263cd8d2208155a45d35ef4434dc967be2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 27 Mar 2016 13:15:57 +0200 Subject: add safeguard --- mitmproxy/protocol/http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index 22e71422..56c5f9ea 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -231,7 +231,8 @@ class HttpLayer(Layer): six.reraise(ProtocolException, ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: - flow.live = False + if flow: + flow.live = False def get_request_from_client(self): request = self.read_request() -- cgit v1.2.3 From ef3d24e8c84dac705a4d1f0bacdc64fec7bffe22 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Sun, 27 Mar 2016 21:42:52 -0700 Subject: locust_task re-use locust_code. --- mitmproxy/flow_export.py | 65 +++++++------------------------------- test/mitmproxy/test_flow_export.py | 29 ++++++++--------- 2 files changed, 25 insertions(+), 69 deletions(-) diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index a9fd5cde..e2d46d93 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -103,10 +103,10 @@ def locust_code(flow): class UserBehavior(TaskSet): def on_start(self): ''' on_start is called when a Locust start before any task is scheduled ''' - self.flow() + self.{name}() @task() - def flow(self): + def {name}(self): url = '{url}' {headers}{params}{data} self.response = self.client.request( @@ -121,10 +121,12 @@ def locust_code(flow): task_set = UserBehavior min_wait = 1000 max_wait = 3000 +""").strip() - """).strip() components = map(lambda x: quote(x, safe=""), flow.request.path_components) + file_name = "_".join(components) + name = re.sub('\W|^(?=\d)', '_', file_name) url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) args = "" @@ -146,6 +148,7 @@ def locust_code(flow): args += "\n data=data," code = code.format( + name=name, url=url, headers=headers, params=params, @@ -163,55 +166,9 @@ def locust_code(flow): def locust_task(flow): - code = dedent(""" - @task() - def {name}(self): - url = '{url}' - {headers}{params}{data} - self.response = self.client.request( - method='{method}', - url=url,{args} - ) - """).strip() - - components = map(lambda x: quote(x, safe=""), flow.request.path_components) - file_name = "_".join(components) - name = re.sub('\W|^(?=\d)','_', file_name) - url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components) - - args = "" - headers = "" - if flow.request.headers: - lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]] - headers += "\n headers = {\n%s }\n" % "".join(lines) - args += "\n headers=headers," - - params = "" - if flow.request.query: - lines = [" '%s': '%s',\n" % (k, v) for k, v in flow.request.query] - params = "\n params = {\n%s }\n" % "".join(lines) - args += "\n params=params," + code = locust_code(flow) + start_task = len(code.split('@task')[0]) - 4 + end_task = -19 - len(code.split('### Additional')[1]) + task_code = code[start_task:end_task] - data = "" - if flow.request.body: - data = "\n data = '''%s'''\n" % flow.request.body - args += "\n data=data," - - code = code.format( - name=name, - url=url, - headers=headers, - params=params, - data=data, - method=flow.request.method, - args=args, - ) - - host = flow.request.scheme + "://" + flow.request.host - code = code.replace(host, "' + self.locust.host + '") - code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) + '") - code = code.replace(quote(host), "' + quote(self.locust.host) + '") - - code = "\n".join(" " + i for i in code.splitlines()) - - return code + return task_code diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 6c9a6756..d937135f 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -188,10 +188,10 @@ from locust import HttpLocust, TaskSet, task class UserBehavior(TaskSet): def on_start(self): ''' on_start is called when a Locust start before any task is scheduled ''' - self.flow() + self.path() @task() - def flow(self): + def path(self): url = '' + self.locust.host + '/path' headers = { @@ -226,10 +226,10 @@ from locust import HttpLocust, TaskSet, task class UserBehavior(TaskSet): def on_start(self): ''' on_start is called when a Locust start before any task is scheduled ''' - self.flow() + self.path() @task() - def flow(self): + def path(self): url = '' + self.locust.host + '/path' data = '''content''' @@ -261,10 +261,10 @@ from locust import HttpLocust, TaskSet, task class UserBehavior(TaskSet): def on_start(self): ''' on_start is called when a Locust start before any task is scheduled ''' - self.flow() + self.path() @task() - def flow(self): + def path(self): url = '' + self.locust.host + '/path' headers = { @@ -312,13 +312,13 @@ class TestExportLocustTask(): 'header': 'qvalue', 'content-length': '7', } - + self.response = self.client.request( method='GET', url=url, headers=headers, ) - """.strip() + """.strip() + '\n' assert flow_export.locust_task(flow) == result @@ -330,14 +330,13 @@ class TestExportLocustTask(): url = '' + self.locust.host + '/path' data = '''content''' - + self.response = self.client.request( method='POST', url=url, data=data, ) - - """.strip() + """.strip() + '\n' assert flow_export.locust_task(flow) == result @@ -353,13 +352,13 @@ class TestExportLocustTask(): 'header': 'qvalue', 'content-length': '7', } - + params = { 'query': 'param', } - + data = '''content''' - + self.response = self.client.request( method='PATCH', url=url, @@ -367,6 +366,6 @@ class TestExportLocustTask(): params=params, data=data, ) - """.strip() + """.strip() + '\n' assert flow_export.locust_task(flow) == result -- cgit v1.2.3 From e56198ae7cf5de1d00c799133d3257434072998e Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Mon, 28 Mar 2016 09:51:06 -0700 Subject: cleaner target url --- mitmproxy/flow_export.py | 1 + test/mitmproxy/test_flow_export.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mitmproxy/flow_export.py b/mitmproxy/flow_export.py index e2d46d93..e2ba7161 100644 --- a/mitmproxy/flow_export.py +++ b/mitmproxy/flow_export.py @@ -161,6 +161,7 @@ def locust_code(flow): code = code.replace(host, "' + self.locust.host + '") code = code.replace(quote_plus(host), "' + quote_plus(self.locust.host) + '") code = code.replace(quote(host), "' + quote(self.locust.host) + '") + code = code.replace("'' + ", "") return code diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index d937135f..ae553685 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -192,7 +192,7 @@ class UserBehavior(TaskSet): @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -230,7 +230,7 @@ class UserBehavior(TaskSet): @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' data = '''content''' @@ -265,7 +265,7 @@ class UserBehavior(TaskSet): @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -306,7 +306,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' headers = { 'header': 'qvalue', @@ -327,7 +327,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' data = '''content''' @@ -346,7 +346,7 @@ class TestExportLocustTask(): result = ' ' + """ @task() def path(self): - url = '' + self.locust.host + '/path' + url = self.locust.host + '/path' headers = { 'header': 'qvalue', -- cgit v1.2.3 From 6d16f44ab723ef23a633b175c1cb4575919b07c4 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Mon, 28 Mar 2016 10:03:26 -0700 Subject: Merge with master --- docs/features/responsestreaming.rst | 3 +-- mitmproxy/console/common.py | 7 +++---- mitmproxy/console/flowview.py | 6 +++--- mitmproxy/dump.py | 5 ++--- mitmproxy/flow.py | 4 ++-- mitmproxy/models/http.py | 10 ++++------ mitmproxy/protocol/http.py | 11 ++++++----- mitmproxy/script/script.py | 2 +- mitmproxy/web/app.py | 5 +---- netlib/http/__init__.py | 4 ++-- netlib/http/authentication.py | 2 +- netlib/http/headers.py | 6 +++--- netlib/http/http1/assemble.py | 10 ++++------ netlib/http/message.py | 8 +++----- netlib/http/request.py | 28 ++++++++++++++-------------- netlib/http/response.py | 8 ++++---- pathod/pathoc.py | 4 ++-- pathod/pathoc_cmdline.py | 2 +- pathod/pathod.py | 2 +- pathod/pathod_cmdline.py | 2 +- pathod/utils.py | 2 +- setup.cfg | 3 +-- test/mitmproxy/test_dump.py | 7 +++---- test/mitmproxy/test_flow.py | 6 +++--- test/mitmproxy/test_flow_export.py | 23 +++++++++++++++++++++++ test/mitmproxy/test_server.py | 6 +++--- test/netlib/http/http1/test_assemble.py | 6 +++--- 27 files changed, 96 insertions(+), 86 deletions(-) diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 8975c1f8..9dc27bf4 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -48,8 +48,7 @@ Implementation Details ---------------------- When response streaming is enabled, portions of the code which would have otherwise performed -changes on the response body will see an empty response body instead -(:py:data:`netlib.http.CONTENT_MISSING`). Any modifications will be ignored. +changes on the response body will see an empty response body. Any modifications will be ignored. Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a ``Transfer-Encoding: chunked`` header, the response will be streamed one chunk at a time. diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index 27e6b6db..141735ef 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -4,7 +4,6 @@ import urwid import urwid.util import os -from netlib.http import CONTENT_MISSING import netlib.utils from .. import utils @@ -256,7 +255,7 @@ def copy_flow_format_data(part, scope, flow): else: data = "" if scope in ("q", "a"): - if flow.request.content is None or flow.request.content == CONTENT_MISSING: + if flow.request.content is None: return None, "Request content is missing" with decoded(flow.request): if part == "h": @@ -269,7 +268,7 @@ def copy_flow_format_data(part, scope, flow): # Add padding between request and response data += "\r\n" * 2 if scope in ("s", "a") and flow.response: - if flow.response.content is None or flow.response.content == CONTENT_MISSING: + if flow.response.content is None: return None, "Response content is missing" with decoded(flow.response): if part == "h": @@ -420,7 +419,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False): if f.response: if f.response.content: contentdesc = netlib.utils.pretty_size(len(f.response.content)) - elif f.response.content == CONTENT_MISSING: + elif f.response.content is None: contentdesc = "[content missing]" else: contentdesc = "[no content]" diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index ba2aad6a..b761a924 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -7,7 +7,7 @@ import math import urwid from netlib import odict -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from . import common, grideditor, signals, searchable, tabs from . import flowdetailview from .. import utils, controller, contentviews @@ -169,7 +169,7 @@ class FlowView(tabs.Tabs): self.show() def content_view(self, viewmode, message): - if message.content == CONTENT_MISSING: + if message.content is None: msg, body = "", [urwid.Text([("error", "[content missing]")])] return msg, body else: @@ -510,7 +510,7 @@ class FlowView(tabs.Tabs): def delete_body(self, t): if t == "m": - val = CONTENT_MISSING + val = None else: val = None if self.tab_offset == TAB_REQ: diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 7b4609b4..631e4d2e 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -5,7 +5,6 @@ import click import itertools from netlib import tcp -from netlib.http import CONTENT_MISSING import netlib.utils from . import flow, filt, contentviews from .exceptions import ContentViewException @@ -180,7 +179,7 @@ class DumpMaster(flow.FlowMaster): ) self.echo(headers, indent=4) if self.o.flow_detail >= 3: - if message.content == CONTENT_MISSING: + if message.content is None: self.echo("(content missing)", indent=4) elif message.content: self.echo("") @@ -283,7 +282,7 @@ class DumpMaster(flow.FlowMaster): code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) reason = click.style(flow.response.reason, fg=code_color, bold=True) - if flow.response.content == CONTENT_MISSING: + if flow.response.content is None: size = "(content missing)" else: size = netlib.utils.pretty_size(len(flow.response.content)) diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index d656bc4d..fbf102b5 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -16,7 +16,7 @@ from six.moves import urllib from netlib import wsgi from netlib.exceptions import HttpException -from netlib.http import CONTENT_MISSING, Headers, http1 +from netlib.http import Headers, http1 from . import controller, tnetstring, filt, script, version, flow_format_compat from .onboarding import app from .proxy.config import HostMatcher @@ -942,7 +942,7 @@ class FlowMaster(controller.Master): return "Can't replay live request." if f.intercepted: return "Can't replay while intercepting..." - if f.request.content == CONTENT_MISSING: + if f.request.content is None: return "Can't replay request with missing content..." if f.request: f.backup() diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index a80e11b0..40460182 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -84,9 +84,8 @@ class HTTPRequest(MessageMixin, Request): headers: Headers object - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. + content: Content of the request, the value is None if there is content + associated, but not present. form_in: The request form which mitmproxy has received. The following values are possible: @@ -226,9 +225,8 @@ class HTTPResponse(MessageMixin, Response): headers: Headers object - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. + content: Content of the response, the value is None if there is content + associated, but not present. timestamp_start: Timestamp indicating when request transmission started diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index a2745eac..56c5f9ea 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -6,7 +6,7 @@ import six from netlib import tcp from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException -from netlib.http import Headers, CONTENT_MISSING +from netlib.http import Headers from h2.exceptions import H2Error @@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer): yield "this is a generator" # pragma: no cover def send_response(self, response): - if response.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if response.content is None: + raise HttpException("Cannot assemble flow with missing content") self.send_response_headers(response) self.send_response_body(response, [response.content]) @@ -231,7 +231,8 @@ class HttpLayer(Layer): six.reraise(ProtocolException, ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: - flow.live = False + if flow: + flow.live = False def get_request_from_client(self): request = self.read_request() @@ -318,7 +319,7 @@ class HttpLayer(Layer): raise Kill() if flow.response.stream: - flow.response.data.content = CONTENT_MISSING + flow.response.data.content = None else: flow.response.data.content = b"".join(self.read_response_body( flow.request, diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py index f142daca..5a8334c4 100644 --- a/mitmproxy/script/script.py +++ b/mitmproxy/script/script.py @@ -47,7 +47,7 @@ class Script(object): if os.name == "nt": # pragma: no cover backslashes = shlex.split(command, posix=False)[0].count("\\") command = command.replace("\\", "\\\\", backslashes) - args = shlex.split(command) # pragma: nocover + args = shlex.split(command) # pragma: no cover args[0] = os.path.expanduser(args[0]) if not os.path.exists(args[0]): raise ScriptException( diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 2cac2ab9..cf3c8bdd 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -8,7 +8,6 @@ import logging import json import base64 -from netlib.http import CONTENT_MISSING from .. import version, filt @@ -26,10 +25,8 @@ def _strip_content(flow_state): continue if message["content"]: message["contentLength"] = len(message["content"]) - elif message["content"] == CONTENT_MISSING: - message["contentLength"] = None else: - message["contentLength"] = 0 + message["contentLength"] = None del message["content"] if "backup" in flow_state: diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index fd632cd5..917080f7 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division from .request import Request from .response import Response from .headers import Headers -from .message import decoded, CONTENT_MISSING +from .message import decoded from . import http1, http2 __all__ = [ "Request", "Response", "Headers", - "decoded", "CONTENT_MISSING", + "decoded", "http1", "http2", ] diff --git a/netlib/http/authentication.py b/netlib/http/authentication.py index d769abe5..6db70fdd 100644 --- a/netlib/http/authentication.py +++ b/netlib/http/authentication.py @@ -140,7 +140,7 @@ class AuthAction(Action): authenticator = BasicProxyAuth(passman, "mitmproxy") setattr(namespace, self.dest, authenticator) - def getPasswordManager(self, s): # pragma: nocover + def getPasswordManager(self, s): # pragma: no cover raise NotImplementedError() diff --git a/netlib/http/headers.py b/netlib/http/headers.py index 78404796..9b8fdae4 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, print_function, division import copy try: from collections.abc import MutableMapping -except ImportError: # pragma: nocover +except ImportError: # pragma: no cover from collections import MutableMapping # Workaround for Python < 3.3 @@ -16,7 +16,7 @@ import six from netlib.utils import always_byte_args, always_bytes, Serializable -if six.PY2: # pragma: nocover +if six.PY2: # pragma: no cover _native = lambda x: x _always_bytes = lambda x: x _always_byte_args = lambda x: x @@ -106,7 +106,7 @@ class Headers(MutableMapping, Serializable): else: return b"" - if six.PY2: # pragma: nocover + if six.PY2: # pragma: no cover __str__ = __bytes__ @_always_byte_args diff --git a/netlib/http/http1/assemble.py b/netlib/http/http1/assemble.py index 785ee8d3..f06ad5a1 100644 --- a/netlib/http/http1/assemble.py +++ b/netlib/http/http1/assemble.py @@ -3,12 +3,10 @@ from __future__ import absolute_import, print_function, division from ... import utils import itertools from ...exceptions import HttpException -from .. import CONTENT_MISSING - def assemble_request(request): - if request.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if request.content is None: + raise HttpException("Cannot assemble flow with missing content") head = assemble_request_head(request) body = b"".join(assemble_body(request.data.headers, [request.data.content])) return head + body @@ -21,8 +19,8 @@ def assemble_request_head(request): def assemble_response(response): - if response.content == CONTENT_MISSING: - raise HttpException("Cannot assemble flow with CONTENT_MISSING") + if response.content is None: + raise HttpException("Cannot assemble flow with missing content") head = assemble_response_head(response) body = b"".join(assemble_body(response.data.headers, [response.data.content])) return head + body diff --git a/netlib/http/message.py b/netlib/http/message.py index e3d8ce37..b6d846ba 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -7,9 +7,7 @@ import six from .headers import Headers from .. import encoding, utils -CONTENT_MISSING = 0 - -if six.PY2: # pragma: nocover +if six.PY2: # pragma: no cover _native = lambda x: x _always_bytes = lambda x: x else: @@ -182,12 +180,12 @@ class Message(utils.Serializable): # Legacy @property - def body(self): # pragma: nocover + def body(self): # pragma: no cover warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning) return self.content @body.setter - def body(self, body): # pragma: nocover + def body(self, body): # pragma: no cover warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning) self.content = body diff --git a/netlib/http/request.py b/netlib/http/request.py index 1a9b6f18..d35c1874 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -100,7 +100,7 @@ class Request(Message): Setting the host attribute also updates the host header, if present. """ - if six.PY2: # pragma: nocover + if six.PY2: # pragma: no cover return self.data.host if not self.data.host: @@ -324,59 +324,59 @@ class Request(Message): # Legacy - def get_cookies(self): # pragma: nocover + def get_cookies(self): # pragma: no cover warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) return self.cookies - def set_cookies(self, odict): # pragma: nocover + def set_cookies(self, odict): # pragma: no cover warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) self.cookies = odict - def get_query(self): # pragma: nocover + def get_query(self): # pragma: no cover warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning) return self.query or ODict([]) - def set_query(self, odict): # pragma: nocover + def set_query(self, odict): # pragma: no cover warnings.warn(".set_query is deprecated, use .query instead.", DeprecationWarning) self.query = odict - def get_path_components(self): # pragma: nocover + def get_path_components(self): # pragma: no cover warnings.warn(".get_path_components is deprecated, use .path_components instead.", DeprecationWarning) return self.path_components - def set_path_components(self, lst): # pragma: nocover + def set_path_components(self, lst): # pragma: no cover warnings.warn(".set_path_components is deprecated, use .path_components instead.", DeprecationWarning) self.path_components = lst - def get_form_urlencoded(self): # pragma: nocover + def get_form_urlencoded(self): # pragma: no cover warnings.warn(".get_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) return self.urlencoded_form or ODict([]) - def set_form_urlencoded(self, odict): # pragma: nocover + def set_form_urlencoded(self, odict): # pragma: no cover warnings.warn(".set_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) self.urlencoded_form = odict - def get_form_multipart(self): # pragma: nocover + def get_form_multipart(self): # pragma: no cover warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning) return self.multipart_form or ODict([]) @property - def form_in(self): # pragma: nocover + def form_in(self): # pragma: no cover warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning) return self.first_line_format @form_in.setter - def form_in(self, form_in): # pragma: nocover + def form_in(self, form_in): # pragma: no cover warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning) self.first_line_format = form_in @property - def form_out(self): # pragma: nocover + def form_out(self): # pragma: no cover warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning) return self.first_line_format @form_out.setter - def form_out(self, form_out): # pragma: nocover + def form_out(self, form_out): # pragma: no cover warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning) self.first_line_format = form_out diff --git a/netlib/http/response.py b/netlib/http/response.py index 8f4d6215..da2c8142 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -97,20 +97,20 @@ class Response(Message): # Legacy - def get_cookies(self): # pragma: nocover + def get_cookies(self): # pragma: no cover warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) return self.cookies - def set_cookies(self, odict): # pragma: nocover + def set_cookies(self, odict): # pragma: no cover warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) self.cookies = odict @property - def msg(self): # pragma: nocover + def msg(self): # pragma: no cover warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning) return self.reason @msg.setter - def msg(self, reason): # pragma: nocover + def msg(self, reason): # pragma: no cover warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning) self.reason = reason diff --git a/pathod/pathoc.py b/pathod/pathoc.py index 86661f98..0e6d3ca7 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -208,7 +208,7 @@ class Pathoc(tcp.TCPClient): self.ws_framereader = None if self.use_http2: - if not tcp.HAS_ALPN: # pragma: nocover + if not tcp.HAS_ALPN: # pragma: no cover log.write_raw( self.fp, "HTTP/2 requires ALPN support. " @@ -458,7 +458,7 @@ class Pathoc(tcp.TCPClient): # TODO: do something -def main(args): # pragma: nocover +def main(args): # pragma: no cover memo = set([]) trycount = 0 p = None diff --git a/pathod/pathoc_cmdline.py b/pathod/pathoc_cmdline.py index bf827a9a..b59704f3 100644 --- a/pathod/pathoc_cmdline.py +++ b/pathod/pathoc_cmdline.py @@ -221,6 +221,6 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): return args -def go_pathoc(): # pragma: nocover +def go_pathoc(): # pragma: no cover args = args_pathoc(sys.argv) pathoc.main(args) diff --git a/pathod/pathod.py b/pathod/pathod.py index 55e75074..b80da887 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -430,7 +430,7 @@ class Pathod(tcp.TCPServer): return self.log -def main(args): # pragma: nocover +def main(args): # pragma: no cover ssloptions = SSLOptions( cn=args.cn, confdir=args.confdir, diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py index c9272249..1f972a49 100644 --- a/pathod/pathod_cmdline.py +++ b/pathod/pathod_cmdline.py @@ -226,6 +226,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): return args -def go_pathod(): # pragma: nocover +def go_pathod(): # pragma: no cover args = args_pathod(sys.argv) pathod.main(args) diff --git a/pathod/utils.py b/pathod/utils.py index a1109a3c..1e5bd9a4 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -98,7 +98,7 @@ class Data(object): data = Data(__name__) -def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: nocover +def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover try: pid = os.fork() if pid > 0: diff --git a/setup.cfg b/setup.cfg index 96b23026..7849ca07 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,7 @@ ignore = E251 [pytest] testpaths = test -addopts = --timeout 30 -s +addopts = --timeout 60 --capture=no [coverage:run] branch = True @@ -19,6 +19,5 @@ omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py [coverage:report] show_missing = True exclude_lines = - pragma: nocover pragma: no cover raise NotImplementedError() diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 2228a732..7e772881 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -4,7 +4,6 @@ from mitmproxy.exceptions import ContentViewException from mitmproxy.models import HTTPResponse import netlib.tutils -from netlib.http import CONTENT_MISSING from mitmproxy import dump, flow from mitmproxy.proxy import Log @@ -38,7 +37,7 @@ def test_strfuncs(): flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=CONTENT_MISSING) + flow.response = netlib.tutils.tresp(content=None) flow.response.is_replay = True flow.response.status_code = 300 m.echo_flow(flow) @@ -104,10 +103,10 @@ class TestDumpMaster: o = dump.Options(flow_detail=3) m = dump.DumpMaster(None, o, outfile=cs) f = tutils.tflow() - f.request.content = CONTENT_MISSING + f.request.content = None m.handle_request(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp()) - f.response.content = CONTENT_MISSING + f.response.content = None m.handle_response(f) assert "content missing" in cs.getvalue() diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 33d2b8f9..2353935b 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -8,7 +8,7 @@ import mock import netlib.utils from netlib import odict -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from mitmproxy import filt, controller, tnetstring, flow from mitmproxy.models import Error from mitmproxy.models import Flow @@ -465,7 +465,7 @@ class TestFlow(object): def test_replace_no_content(self): f = tutils.tflow() - f.request.content = CONTENT_MISSING + f.request.content = None assert f.replace("foo", "bar") == 0 def test_replace(self): @@ -751,7 +751,7 @@ class TestFlowMaster: s = flow.State() fm = flow.FlowMaster(None, s) f = tutils.tflow(resp=True) - f.request.content = CONTENT_MISSING + f.request.content = None assert "missing" in fm.replay_request(f) f.intercepted = True diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index ae553685..d4b25543 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -1,3 +1,4 @@ +import json from textwrap import dedent import netlib.tutils @@ -178,6 +179,7 @@ class TestRawRequest(): """).strip() assert flow_export.raw_request(flow) == result + class TestExportLocustCode(): def test_get(self): @@ -369,3 +371,24 @@ class TestExportLocustTask(): """.strip() + '\n' assert flow_export.locust_task(flow) == result + + +class TestIsJson(): + + def test_empty(self): + assert flow_export.is_json(None, None) == False + + def test_json_type(self): + headers = Headers(content_type="application/json") + assert flow_export.is_json(headers, "foobar") == False + + def test_valid(self): + headers = Headers(content_type="application/foobar") + j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + assert j == False + + def test_valid(self): + headers = Headers(content_type="application/json") + j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + assert isinstance(j, dict) + diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 26e53e8a..dc72f032 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -8,7 +8,7 @@ from netlib.tcp import Address import netlib.tutils from netlib import tcp, http, socks from netlib.certutils import SSLCert -from netlib.http import authentication, CONTENT_MISSING, http1 +from netlib.http import authentication, http1 from netlib.tutils import raises from pathod import pathoc, pathod @@ -281,7 +281,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.pathod("200:b@3k") assert self.master.state.view[-1].response.stream - assert self.master.state.view[-1].response.content == CONTENT_MISSING + assert self.master.state.view[-1].response.content is None self.master.set_stream_large_bodies(None) def test_stream_modify(self): @@ -816,7 +816,7 @@ class MasterIncomplete(tservers.TestMaster): def handle_request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) - resp.content = CONTENT_MISSING + resp.content = None f.reply(resp) diff --git a/test/netlib/http/http1/test_assemble.py b/test/netlib/http/http1/test_assemble.py index 31a62438..8dcbae8e 100644 --- a/test/netlib/http/http1/test_assemble.py +++ b/test/netlib/http/http1/test_assemble.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, print_function, division from netlib.exceptions import HttpException -from netlib.http import CONTENT_MISSING, Headers +from netlib.http import Headers from netlib.http.http1.assemble import ( assemble_request, assemble_request_head, assemble_response, assemble_response_head, _assemble_request_line, _assemble_request_headers, @@ -20,7 +20,7 @@ def test_assemble_request(): ) with raises(HttpException): - assemble_request(treq(content=CONTENT_MISSING)) + assemble_request(treq(content=None)) def test_assemble_request_head(): @@ -41,7 +41,7 @@ def test_assemble_response(): ) with raises(HttpException): - assemble_response(tresp(content=CONTENT_MISSING)) + assemble_response(tresp(content=None)) def test_assemble_response_head(): -- cgit v1.2.3 From 37483e228fb06a8b8d15a32178eb24e59b9e67c5 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Mon, 28 Mar 2016 10:06:02 -0700 Subject: Merge remote-tracking branch 'mitmproxy/master' Merge with master --- test/mitmproxy/test_flow_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index d4b25543..c1b14da9 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -391,4 +391,3 @@ class TestIsJson(): headers = Headers(content_type="application/json") j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') assert isinstance(j, dict) - -- cgit v1.2.3 From cd2ef2fe139a3c9c03814bb2db41bccae36583a5 Mon Sep 17 00:00:00 2001 From: Zohar Lorberbaum Date: Mon, 28 Mar 2016 10:17:58 -0700 Subject: merge --- test/mitmproxy/test_flow_export.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index c1b14da9..75a8090f 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -179,7 +179,6 @@ class TestRawRequest(): """).strip() assert flow_export.raw_request(flow) == result - class TestExportLocustCode(): def test_get(self): -- cgit v1.2.3