aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--examples/har_extractor.py2
-rw-r--r--mitmproxy/console/__init__.py2
-rw-r--r--mitmproxy/console/flowdetailview.py110
-rw-r--r--mitmproxy/flow.py26
-rw-r--r--mitmproxy/models/flow.py1
-rw-r--r--mitmproxy/models/http.py89
-rw-r--r--mitmproxy/protocol/http.py18
-rw-r--r--mitmproxy/protocol/http1.py2
-rw-r--r--mitmproxy/protocol/http2.py6
-rw-r--r--mitmproxy/protocol/http_replay.py10
-rw-r--r--mitmproxy/script/concurrent.py27
-rw-r--r--netlib/certutils.py2
-rw-r--r--netlib/http/cookies.py34
-rw-r--r--netlib/http/http2/connections.py8
-rw-r--r--netlib/http/request.py21
-rw-r--r--netlib/http/response.py44
-rw-r--r--pathod/pathoc.py2
-rw-r--r--setup.py11
-rw-r--r--test/mitmproxy/test_flow.py37
-rw-r--r--test/mitmproxy/test_server.py16
-rw-r--r--test/netlib/http/http2/test_connections.py16
-rw-r--r--test/netlib/http/test_cookies.py16
-rw-r--r--test/netlib/http/test_response.py23
23 files changed, 247 insertions, 276 deletions
diff --git a/examples/har_extractor.py b/examples/har_extractor.py
index 15e1ef30..371e2282 100644
--- a/examples/har_extractor.py
+++ b/examples/har_extractor.py
@@ -143,7 +143,7 @@ def response(context, flow):
},
"response": {
"status": flow.response.status_code,
- "statusText": flow.response.msg,
+ "statusText": flow.response.reason,
"httpVersion": flow.response.http_version,
"cookies": format_cookies(flow.response.cookies),
"headers": format_headers(flow.response.headers),
diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py
index 2f52d0b8..9deb26e1 100644
--- a/mitmproxy/console/__init__.py
+++ b/mitmproxy/console/__init__.py
@@ -257,7 +257,7 @@ class ConsoleMaster(flow.FlowMaster):
def load_script(self, command, use_reloader=True):
# We default to using the reloader in the console ui.
- super(ConsoleMaster, self).load_script(command, use_reloader)
+ return super(ConsoleMaster, self).load_script(command, use_reloader)
def sig_add_event(self, sender, e, level):
needed = dict(error=0, info=1, debug=2).get(level, 1)
diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/console/flowdetailview.py
index 8e3a47ae..d338b6bc 100644
--- a/mitmproxy/console/flowdetailview.py
+++ b/mitmproxy/console/flowdetailview.py
@@ -5,7 +5,7 @@ from .. import utils
def maybe_timestamp(base, attr):
- if base and getattr(base, attr):
+ if base is not None and getattr(base, attr):
return utils.format_timestamp_with_milli(getattr(base, attr))
else:
return "active"
@@ -89,65 +89,71 @@ def flowdetails(state, flow):
parts = []
- parts.append(
- [
- "Client conn. established",
- maybe_timestamp(cc, "timestamp_start")
- ]
- )
- parts.append(
- [
- "Server conn. initiated",
- maybe_timestamp(sc, "timestamp_start")
- ]
- )
- parts.append(
- [
- "Server conn. TCP handshake",
- maybe_timestamp(sc, "timestamp_tcp_setup")
- ]
- )
- if sc.ssl_established:
+ if cc is not None and cc.timestamp_start:
parts.append(
[
- "Server conn. SSL handshake",
- maybe_timestamp(sc, "timestamp_ssl_setup")
+ "Client conn. established",
+ maybe_timestamp(cc, "timestamp_start")
]
)
+ if cc.ssl_established:
+ parts.append(
+ [
+ "Client conn. TLS handshake",
+ maybe_timestamp(cc, "timestamp_ssl_setup")
+ ]
+ )
+ if sc is not None and sc.timestamp_start:
parts.append(
[
- "Client conn. SSL handshake",
- maybe_timestamp(cc, "timestamp_ssl_setup")
+ "Server conn. initiated",
+ maybe_timestamp(sc, "timestamp_start")
+ ]
+ )
+ parts.append(
+ [
+ "Server conn. TCP handshake",
+ maybe_timestamp(sc, "timestamp_tcp_setup")
+ ]
+ )
+ if sc.ssl_established:
+ parts.append(
+ [
+ "Server conn. TLS handshake",
+ maybe_timestamp(sc, "timestamp_ssl_setup")
+ ]
+ )
+ if req is not None and req.timestamp_start:
+ parts.append(
+ [
+ "First request byte",
+ maybe_timestamp(req, "timestamp_start")
+ ]
+ )
+ parts.append(
+ [
+ "Request complete",
+ maybe_timestamp(req, "timestamp_end")
+ ]
+ )
+ if resp is not None and resp.timestamp_start:
+ parts.append(
+ [
+ "First response byte",
+ maybe_timestamp(resp, "timestamp_start")
+ ]
+ )
+ parts.append(
+ [
+ "Response complete",
+ maybe_timestamp(resp, "timestamp_end")
]
)
- parts.append(
- [
- "First request byte",
- maybe_timestamp(req, "timestamp_start")
- ]
- )
- parts.append(
- [
- "Request complete",
- maybe_timestamp(req, "timestamp_end")
- ]
- )
- parts.append(
- [
- "First response byte",
- maybe_timestamp(resp, "timestamp_start")
- ]
- )
- parts.append(
- [
- "Response complete",
- maybe_timestamp(resp, "timestamp_end")
- ]
- )
- # sort operations by timestamp
- parts = sorted(parts, key=lambda p: p[1])
+ if parts:
+ # sort operations by timestamp
+ parts = sorted(parts, key=lambda p: p[1])
- text.append(urwid.Text([("head", "Timing:")]))
- text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+ text.append(urwid.Text([("head", "Timing:")]))
+ text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
return searchable.Searchable(state, text)
diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
index fbf102b5..eefe1b5e 100644
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -806,7 +806,7 @@ class FlowMaster(controller.Master):
response.is_replay = True
if self.refresh_server_playback:
response.refresh()
- flow.reply(response)
+ flow.response = response
if self.server_playback.count() == 0:
self.stop_server_playback()
return True
@@ -838,20 +838,21 @@ class FlowMaster(controller.Master):
address=dict(address=(host, port), use_ipv6=False),
clientcert=None,
ssl_established=False,
- timestamp_start=time.time(),
- timestamp_end=time.time(),
- timestamp_ssl_setup=time.time()
+ timestamp_start=None,
+ timestamp_end=None,
+ timestamp_ssl_setup=None
))
s = ServerConnection.from_state(dict(
address=dict(address=(host, port), use_ipv6=False),
+ peer_address=None,
cert=None,
sni=host,
source_address=dict(address=('', 0), use_ipv6=False),
ssl_established=True,
- timestamp_start=time.time(),
- timestamp_tcp_setup=time.time(),
- timestamp_ssl_setup=time.time(),
+ timestamp_start=None,
+ timestamp_tcp_setup=None,
+ timestamp_ssl_setup=None,
timestamp_end=None,
via=None
))
@@ -905,11 +906,11 @@ class FlowMaster(controller.Master):
def load_flows_file(self, path):
path = os.path.expanduser(path)
try:
- f = file(path, "rb")
- freader = FlowReader(f)
+ with open(path, "rb") as f:
+ freader = FlowReader(f)
+ return self.load_flows(freader)
except IOError as v:
raise FlowReadError(v.strerror)
- return self.load_flows(freader)
def process_new_request(self, f):
if self.stickycookie_state:
@@ -924,11 +925,8 @@ class FlowMaster(controller.Master):
if self.server_playback:
pb = self.do_server_playback(f)
- if not pb:
- if self.kill_nonreplay:
+ if not pb and self.kill_nonreplay:
f.kill(self)
- else:
- f.reply()
def process_new_response(self, f):
if self.stickycookie_state:
diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py
index 95fb2b69..45b3b5e0 100644
--- a/mitmproxy/models/flow.py
+++ b/mitmproxy/models/flow.py
@@ -28,7 +28,6 @@ class Error(stateobject.StateObject):
@type msg: str
@type timestamp: float
"""
- self.flow = None # will usually be set by the flow backref mixin
self.msg = msg
self.timestamp = timestamp or utils.timestamp()
diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py
index 11f46611..08fc5e46 100644
--- a/mitmproxy/models/http.py
+++ b/mitmproxy/models/http.py
@@ -1,5 +1,5 @@
from __future__ import (absolute_import, print_function, division)
-from six.moves import http_cookies as Cookie
+import cgi
import copy
import warnings
from email.utils import parsedate_tz, formatdate, mktime_tz
@@ -55,18 +55,13 @@ class HTTPRequest(MessageMixin, Request):
content: Content of the request, the value is None if there is content
associated, but not present.
- form_in: The request form which mitmproxy has received. The following
- values are possible:
+ first_line_format: The request form. The following values are possible:
- - relative (GET /index.html, OPTIONS *) (covers origin form and
- asterisk form)
+ - relative (GET /index.html, OPTIONS *) (origin form or asterisk form)
- absolute (GET http://example.com:80/index.html)
- authority-form (CONNECT example.com:443)
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
- form_out: The request form which mitmproxy will send out to the
- destination
-
timestamp_start: Timestamp indicating when request transmission started
timestamp_end: Timestamp indicating when request transmission ended
@@ -85,7 +80,6 @@ class HTTPRequest(MessageMixin, Request):
content,
timestamp_start=None,
timestamp_end=None,
- form_out=None,
is_replay=False,
stickycookie=False,
stickyauth=False,
@@ -104,7 +98,6 @@ class HTTPRequest(MessageMixin, Request):
timestamp_start,
timestamp_end,
)
- self.form_out = form_out or first_line_format # FIXME remove
# Have this request's cookies been modified by sticky cookies or auth?
self.stickycookie = stickycookie
@@ -142,26 +135,12 @@ class HTTPRequest(MessageMixin, Request):
content=request.data.content,
timestamp_start=request.data.timestamp_start,
timestamp_end=request.data.timestamp_end,
- form_out=(request.form_out if hasattr(request, 'form_out') else None),
)
return req
- @property
- def form_out(self):
- warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
- return self.first_line_format
-
- @form_out.setter
- def form_out(self, value):
- warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
- self.first_line_format = value
-
def __hash__(self):
return id(self)
- def set_auth(self, auth):
- self.data.headers.set_all("Proxy-Authorization", (auth,))
-
class HTTPResponse(MessageMixin, Response):
"""
@@ -224,66 +203,6 @@ class HTTPResponse(MessageMixin, Response):
)
return resp
- def _refresh_cookie(self, c, delta):
- """
- Takes a cookie string c and a time delta in seconds, and returns
- a refreshed cookie string.
- """
- try:
- c = Cookie.SimpleCookie(str(c))
- except Cookie.CookieError:
- raise ValueError("Invalid Cookie")
- for i in c.values():
- if "expires" in i:
- d = parsedate_tz(i["expires"])
- if d:
- d = mktime_tz(d) + delta
- i["expires"] = formatdate(d)
- else:
- # This can happen when the expires tag is invalid.
- # reddit.com sends a an expires tag like this: "Thu, 31 Dec
- # 2037 23:59:59 GMT", which is valid RFC 1123, but not
- # strictly correct according to the cookie spec. Browsers
- # appear to parse this tolerantly - maybe we should too.
- # For now, we just ignore this.
- del i["expires"]
- ret = c.output(header="").strip()
- if not ret:
- raise ValueError("Invalid Cookie")
- return ret
-
- def refresh(self, now=None):
- """
- This fairly complex and heuristic function refreshes a server
- response for replay.
-
- - It adjusts date, expires and last-modified headers.
- - It adjusts cookie expiration.
- """
- if not now:
- now = time.time()
- delta = now - self.timestamp_start
- refresh_headers = [
- "date",
- "expires",
- "last-modified",
- ]
- for i in refresh_headers:
- if i in self.headers:
- d = parsedate_tz(self.headers[i])
- if d:
- new = mktime_tz(d) + delta
- self.headers[i] = formatdate(new)
- c = []
- for set_cookie_header in self.headers.get_all("set-cookie"):
- try:
- refreshed = self._refresh_cookie(set_cookie_header, delta)
- except ValueError:
- refreshed = set_cookie_header
- c.append(refreshed)
- if c:
- self.headers.set_all("set-cookie", c)
-
class HTTPFlow(Flow):
@@ -381,7 +300,7 @@ def make_error_response(status_code, message, headers=None):
</head>
<body>%s</body>
</html>
- """.strip() % (status_code, response, message)
+ """.strip() % (status_code, response, cgi.escape(message))
body = body.encode("utf8", "replace")
if not headers:
diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py
index 56c5f9ea..db24ca92 100644
--- a/mitmproxy/protocol/http.py
+++ b/mitmproxy/protocol/http.py
@@ -167,7 +167,7 @@ class HttpLayer(Layer):
self.validate_request(request)
# Regular Proxy Mode: Handle CONNECT
- if self.mode == "regular" and request.form_in == "authority":
+ if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
@@ -184,7 +184,7 @@ class HttpLayer(Layer):
flow.request = request
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
- flow.request.set_auth(self.config.upstream_auth)
+ self.data.headers["Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(flow)
if not flow.response:
@@ -215,7 +215,7 @@ class HttpLayer(Layer):
return
# Upstream Proxy Mode: Handle CONNECT
- if flow.request.form_in == "authority" and flow.response.status_code == 200:
+ if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
self.handle_upstream_mode_connect(flow.request.copy())
return
@@ -340,7 +340,7 @@ class HttpLayer(Layer):
if self.mode == "regular":
pass # only absolute-form at this point, nothing to do here.
elif self.mode == "upstream":
- if flow.request.form_in == "authority":
+ if flow.request.first_line_format == "authority":
flow.request.scheme = "http" # pseudo value
else:
# Setting request.host also updates the host header, which we want to preserve
@@ -390,7 +390,7 @@ class HttpLayer(Layer):
"""
def validate_request(self, request):
- if request.form_in == "absolute" and request.scheme != "http":
+ if request.first_line_format == "absolute" and request.scheme != "http":
raise HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
@@ -400,14 +400,14 @@ class HttpLayer(Layer):
}
allowed_request_forms = expected_request_forms[self.mode]
- if request.form_in not in allowed_request_forms:
+ if request.first_line_format not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
- " or ".join(allowed_request_forms), request.form_in
+ " or ".join(allowed_request_forms), request.first_line_format
)
raise HttpException(err_message)
- if self.mode == "regular" and request.form_in == "absolute":
- request.form_out = "relative"
+ if self.mode == "regular" and request.first_line_format == "absolute":
+ request.first_line_format = "relative"
def authenticate(self, request):
if self.config.authenticator:
diff --git a/mitmproxy/protocol/http1.py b/mitmproxy/protocol/http1.py
index a4cd8801..940a4c98 100644
--- a/mitmproxy/protocol/http1.py
+++ b/mitmproxy/protocol/http1.py
@@ -54,7 +54,7 @@ class Http1Layer(_HttpTransmissionLayer):
)
read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1
close_connection = request_close or response_close or read_until_eof
- if flow.request.form_in == "authority" and flow.response.status_code == 200:
+ if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Charles Proxy sends a CONNECT response with HTTP/1.0
# and no Content-Length header
diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py
index eba4795e..03d4aefc 100644
--- a/mitmproxy/protocol/http2.py
+++ b/mitmproxy/protocol/http2.py
@@ -303,11 +303,11 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
port = None
if path == '*' or path.startswith("/"):
- form_in = "relative"
+ first_line_format = "relative"
elif method == 'CONNECT': # pragma: no cover
raise NotImplementedError("CONNECT over HTTP/2 is not implemented.")
else: # pragma: no cover
- form_in = "absolute"
+ first_line_format = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = utils.parse_url(path)
@@ -326,7 +326,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
data = b"".join(data)
return HTTPRequest(
- form_in,
+ first_line_format,
method,
scheme,
host,
diff --git a/mitmproxy/protocol/http_replay.py b/mitmproxy/protocol/http_replay.py
index 6316f26c..62f842ce 100644
--- a/mitmproxy/protocol/http_replay.py
+++ b/mitmproxy/protocol/http_replay.py
@@ -30,7 +30,7 @@ class RequestReplayThread(threading.Thread):
def run(self):
r = self.flow.request
- form_out_backup = r.form_out
+ first_line_format_backup = r.first_line_format
try:
self.flow.response = None
@@ -63,9 +63,9 @@ class RequestReplayThread(threading.Thread):
self.config.clientcerts,
sni=self.flow.server_conn.sni
)
- r.form_out = "relative"
+ r.first_line_format = "relative"
else:
- r.form_out = "absolute"
+ r.first_line_format= "absolute"
else:
server_address = (r.host, r.port)
server = ServerConnection(server_address, (self.config.host, 0))
@@ -75,7 +75,7 @@ class RequestReplayThread(threading.Thread):
self.config.clientcerts,
sni=self.flow.server_conn.sni
)
- r.form_out = "relative"
+ r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
@@ -102,4 +102,4 @@ class RequestReplayThread(threading.Thread):
from ..proxy.root_context import Log
self.channel.tell("log", Log(traceback.format_exc(), "error"))
finally:
- r.form_out = form_out_backup
+ r.first_line_format = first_line_format_backup
diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py
index 57ee37de..2f25e78c 100644
--- a/mitmproxy/script/concurrent.py
+++ b/mitmproxy/script/concurrent.py
@@ -8,22 +8,23 @@ import threading
class ReplyProxy(object):
- def __init__(self, original_reply, script_thread):
- self.original_reply = original_reply
+ def __init__(self, reply_func, script_thread):
+ self.reply_func = reply_func
self.script_thread = script_thread
- self._ignore_call = True
- self.lock = threading.Lock()
+ self.master_reply = None
- def __call__(self, *args, **kwargs):
- with self.lock:
- if self._ignore_call:
- self.script_thread.start()
- self._ignore_call = False
- return
- self.original_reply(*args, **kwargs)
+ def __call__(self, *args):
+ if self.master_reply is None:
+ self.master_reply = args
+ self.script_thread.start()
+ return
+ self.reply_func(*args)
+
+ def done(self):
+ self.reply_func(*self.master_reply)
def __getattr__(self, k):
- return getattr(self.original_reply, k)
+ return getattr(self.reply_func, k)
def _handle_concurrent_reply(fn, o, *args, **kwargs):
@@ -34,7 +35,7 @@ def _handle_concurrent_reply(fn, o, *args, **kwargs):
def run():
fn(*args, **kwargs)
# If the script did not call .reply(), we have to do it now.
- reply_proxy()
+ reply_proxy.done()
script_thread = ScriptThread(target=run)
diff --git a/netlib/certutils.py b/netlib/certutils.py
index 616a778e..34e01ed3 100644
--- a/netlib/certutils.py
+++ b/netlib/certutils.py
@@ -386,7 +386,7 @@ class SSLCert(Serializable):
@classmethod
def from_state(cls, state):
- cls.from_pem(state)
+ return cls.from_pem(state)
@classmethod
def from_pem(cls, txt):
diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py
index 18544b5e..caa84ff7 100644
--- a/netlib/http/cookies.py
+++ b/netlib/http/cookies.py
@@ -1,4 +1,6 @@
+from six.moves import http_cookies as Cookie
import re
+from email.utils import parsedate_tz, formatdate, mktime_tz
from .. import odict
@@ -191,3 +193,35 @@ def format_cookie_header(od):
Formats a Cookie header value.
"""
return _format_pairs(od.lst)
+
+
+def refresh_set_cookie_header(c, delta):
+ """
+ Args:
+ c: A Set-Cookie string
+ delta: Time delta in seconds
+ Returns:
+ A refreshed Set-Cookie string
+ """
+ try:
+ c = Cookie.SimpleCookie(str(c))
+ except Cookie.CookieError:
+ raise ValueError("Invalid Cookie")
+ for i in c.values():
+ if "expires" in i:
+ d = parsedate_tz(i["expires"])
+ if d:
+ d = mktime_tz(d) + delta
+ i["expires"] = formatdate(d)
+ else:
+ # This can happen when the expires tag is invalid.
+ # reddit.com sends a an expires tag like this: "Thu, 31 Dec
+ # 2037 23:59:59 GMT", which is valid RFC 1123, but not
+ # strictly correct according to the cookie spec. Browsers
+ # appear to parse this tolerantly - maybe we should too.
+ # For now, we just ignore this.
+ del i["expires"]
+ ret = c.output(header="").strip()
+ if not ret:
+ raise ValueError("Invalid Cookie")
+ return ret
diff --git a/netlib/http/http2/connections.py b/netlib/http/http2/connections.py
index 52fa7193..f900b67c 100644
--- a/netlib/http/http2/connections.py
+++ b/netlib/http/http2/connections.py
@@ -102,15 +102,15 @@ class HTTP2Protocol(object):
port = None
if path == '*' or path.startswith("/"):
- form_in = "relative"
+ first_line_format = "relative"
elif method == 'CONNECT':
- form_in = "authority"
+ first_line_format = "authority"
if ":" in authority:
host, port = authority.split(":", 1)
else:
host = authority
else:
- form_in = "absolute"
+ first_line_format = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = utils.parse_url(path)
scheme = scheme.decode('ascii')
@@ -123,7 +123,7 @@ class HTTP2Protocol(object):
port = int(port)
request = Request(
- form_in,
+ first_line_format,
method.encode('ascii'),
scheme.encode('ascii'),
host.encode('ascii'),
diff --git a/netlib/http/request.py b/netlib/http/request.py
index 07a11969..692ba30f 100644
--- a/netlib/http/request.py
+++ b/netlib/http/request.py
@@ -375,24 +375,3 @@ class Request(Message):
def get_form_multipart(self): # pragma: no cover
warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning)
return self.multipart_form or ODict([])
-
- @property
- def form_in(self): # pragma: no cover
- warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
- return self.first_line_format
-
- @form_in.setter
- def form_in(self, form_in): # pragma: no cover
- warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
- self.first_line_format = form_in
-
- @property
- def form_out(self): # pragma: no cover
- warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
- return self.first_line_format
-
- @form_out.setter
- def form_out(self, form_out): # pragma: no cover
- warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
- self.first_line_format = form_out
-
diff --git a/netlib/http/response.py b/netlib/http/response.py
index 8af3c041..efd7f60a 100644
--- a/netlib/http/response.py
+++ b/netlib/http/response.py
@@ -1,6 +1,8 @@
from __future__ import absolute_import, print_function, division
import warnings
+from email.utils import parsedate_tz, formatdate, mktime_tz
+import time
from . import cookies
from .headers import Headers
@@ -94,6 +96,38 @@ class Response(Message):
values.append(header)
self.headers.set_all("set-cookie", values)
+ def refresh(self, now=None):
+ """
+ This fairly complex and heuristic function refreshes a server
+ response for replay.
+
+ - It adjusts date, expires and last-modified headers.
+ - It adjusts cookie expiration.
+ """
+ if not now:
+ now = time.time()
+ delta = now - self.timestamp_start
+ refresh_headers = [
+ "date",
+ "expires",
+ "last-modified",
+ ]
+ for i in refresh_headers:
+ if i in self.headers:
+ d = parsedate_tz(self.headers[i])
+ if d:
+ new = mktime_tz(d) + delta
+ self.headers[i] = formatdate(new)
+ c = []
+ for set_cookie_header in self.headers.get_all("set-cookie"):
+ try:
+ refreshed = cookies.refresh_set_cookie_header(set_cookie_header, delta)
+ except ValueError:
+ refreshed = set_cookie_header
+ c.append(refreshed)
+ if c:
+ self.headers.set_all("set-cookie", c)
+
# Legacy
def get_cookies(self): # pragma: no cover
@@ -103,13 +137,3 @@ class Response(Message):
def set_cookies(self, odict): # pragma: no cover
warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning)
self.cookies = odict
-
- @property
- def msg(self): # pragma: no cover
- warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
- return self.reason
-
- @msg.setter
- def msg(self, reason): # pragma: no cover
- warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
- self.reason = reason
diff --git a/pathod/pathoc.py b/pathod/pathoc.py
index 0e6d3ca7..f55a2eda 100644
--- a/pathod/pathoc.py
+++ b/pathod/pathoc.py
@@ -425,7 +425,7 @@ class Pathoc(tcp.TCPClient):
finally:
if resp:
lg("<< %s %s: %s bytes" % (
- resp.status_code, utils.xrepr(resp.msg), len(resp.content)
+ resp.status_code, utils.xrepr(resp.reason), len(resp.content)
))
if resp.status_code in self.ignorecodes:
lg.suppress()
diff --git a/setup.py b/setup.py
index 4465fe3c..0d784818 100644
--- a/setup.py
+++ b/setup.py
@@ -69,9 +69,10 @@ setup(
"Flask>=0.10.1, <0.11",
"h2>=2.1.2, <3.0",
"hpack>=2.1.0, <3.0",
- "html2text==2016.1.8",
+ "html2text>=2016.1.8, <=2016.4.2",
"hyperframe>=3.2.0, <4.0",
- "Pillow>=3.1, <3.2",
+ "lxml>=3.5.0, <3.7",
+ "Pillow>=3.2, <3.3",
"passlib>=1.6.5, <1.7",
"pyasn1>=0.1.9, <0.2",
"pyOpenSSL>=16.0, <17.0",
@@ -85,11 +86,9 @@ setup(
],
extras_require={
':sys_platform == "win32"': [
- "lxml==3.5.0", # there are no Windows wheels for newer versions, so we pin this.
"pydivert>=0.0.7, <0.1",
],
':sys_platform != "win32"': [
- "lxml>=3.5.0, <3.7",
],
# Do not use a range operator here: https://bitbucket.org/pypa/setuptools/issues/380
# Ubuntu Trusty and other still ship with setuptools < 17.1
@@ -104,7 +103,7 @@ setup(
"pytest-cov>=2.2.1, <2.3",
"pytest-timeout>=1.0.0, <1.1",
"pytest-xdist>=1.14, <1.15",
- "sphinx>=1.3.5, <1.4",
+ "sphinx>=1.3.5, <1.5",
"sphinx-autobuild>=0.5.2, <0.7",
"sphinxcontrib-documentedlist>=0.3.0, <0.4",
],
@@ -116,7 +115,7 @@ setup(
'examples': [
"beautifulsoup4>=4.4.1, <4.5",
"harparser>=0.2, <0.3",
- "pytz>=2015.07.0, <=2016.1",
+ "pytz>=2015.07.0, <=2016.3",
]
}
)
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index 2353935b..1fadb23d 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -809,6 +809,11 @@ class TestFlowMaster:
assert s.flow_count() == 2
assert s.index(f2) == 1
+ def test_create_flow(self):
+ s = flow.State()
+ fm = flow.FlowMaster(None, s)
+ assert fm.create_request("GET", "http", "example.com", 80, "/")
+
def test_all(self):
s = flow.State()
fm = flow.FlowMaster(None, s)
@@ -1146,38 +1151,6 @@ class TestResponse:
resp2 = resp.copy()
assert resp2.get_state() == resp.get_state()
- def test_refresh(self):
- r = HTTPResponse.wrap(netlib.tutils.tresp())
- n = time.time()
- r.headers["date"] = email.utils.formatdate(n)
- pre = r.headers["date"]
- r.refresh(n)
- assert pre == r.headers["date"]
- r.refresh(n + 60)
-
- d = email.utils.parsedate_tz(r.headers["date"])
- d = email.utils.mktime_tz(d)
- # Weird that this is not exact...
- assert abs(60 - (d - n)) <= 1
-
- r.headers["set-cookie"] = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
- r.refresh()
-
- def test_refresh_cookie(self):
- r = HTTPResponse.wrap(netlib.tutils.tresp())
-
- # Invalid expires format, sent to us by Reddit.
- c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/"
- assert r._refresh_cookie(c, 60)
-
- c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
- assert "00:21:38" in r._refresh_cookie(c, 60)
-
- # https://github.com/mitmproxy/mitmproxy/issues/773
- c = ">=A"
- with tutils.raises(ValueError):
- r._refresh_cookie(c, 60)
-
def test_replace(self):
r = HTTPResponse.wrap(netlib.tutils.tresp())
r.headers["Foo"] = "fOo"
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index dc72f032..0d56e7ea 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -973,22 +973,22 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
assert not self.chain[1].tmaster.state.flows[0].response # killed
assert self.chain[1].tmaster.state.flows[1].response
- assert self.proxy.tmaster.state.flows[0].request.form_in == "authority"
- assert self.proxy.tmaster.state.flows[1].request.form_in == "relative"
+ assert self.proxy.tmaster.state.flows[0].request.first_line_format == "authority"
+ assert self.proxy.tmaster.state.flows[1].request.first_line_format == "relative"
assert self.chain[0].tmaster.state.flows[
- 0].request.form_in == "authority"
+ 0].request.first_line_format == "authority"
assert self.chain[0].tmaster.state.flows[
- 1].request.form_in == "relative"
+ 1].request.first_line_format == "relative"
assert self.chain[0].tmaster.state.flows[
- 2].request.form_in == "authority"
+ 2].request.first_line_format == "authority"
assert self.chain[0].tmaster.state.flows[
- 3].request.form_in == "relative"
+ 3].request.first_line_format == "relative"
assert self.chain[1].tmaster.state.flows[
- 0].request.form_in == "relative"
+ 0].request.first_line_format == "relative"
assert self.chain[1].tmaster.state.flows[
- 1].request.form_in == "relative"
+ 1].request.first_line_format == "relative"
req = p.request("get:'/p/418:b\"content2\"'")
diff --git a/test/netlib/http/http2/test_connections.py b/test/netlib/http/http2/test_connections.py
index c067d487..7b003067 100644
--- a/test/netlib/http/http2/test_connections.py
+++ b/test/netlib/http/http2/test_connections.py
@@ -325,7 +325,7 @@ class TestReadRequestRelative(tservers.ServerTestBase):
ssl = True
- def test_asterisk_form_in(self):
+ def test_asterisk_form(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
c.connect()
c.convert_to_ssl()
@@ -334,7 +334,7 @@ class TestReadRequestRelative(tservers.ServerTestBase):
req = protocol.read_request(NotImplemented)
- assert req.form_in == "relative"
+ assert req.first_line_format == "relative"
assert req.method == "OPTIONS"
assert req.path == "*"
@@ -348,7 +348,7 @@ class TestReadRequestAbsolute(tservers.ServerTestBase):
ssl = True
- def test_absolute_form_in(self):
+ def test_absolute_form(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
c.connect()
c.convert_to_ssl()
@@ -357,7 +357,7 @@ class TestReadRequestAbsolute(tservers.ServerTestBase):
req = protocol.read_request(NotImplemented)
- assert req.form_in == "absolute"
+ assert req.first_line_format == "absolute"
assert req.scheme == "http"
assert req.host == "address"
assert req.port == 22
@@ -382,13 +382,13 @@ class TestReadRequestConnect(tservers.ServerTestBase):
protocol.connection_preface_performed = True
req = protocol.read_request(NotImplemented)
- assert req.form_in == "authority"
+ assert req.first_line_format == "authority"
assert req.method == "CONNECT"
assert req.host == "address"
assert req.port == 22
req = protocol.read_request(NotImplemented)
- assert req.form_in == "authority"
+ assert req.first_line_format == "authority"
assert req.method == "CONNECT"
assert req.host == "example.com"
assert req.port == 443
@@ -417,7 +417,7 @@ class TestReadResponse(tservers.ServerTestBase):
assert resp.http_version == "HTTP/2.0"
assert resp.status_code == 200
- assert resp.msg == ''
+ assert resp.reason == ''
assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
assert resp.content == b'foobar'
assert resp.timestamp_end
@@ -444,7 +444,7 @@ class TestReadEmptyResponse(tservers.ServerTestBase):
assert resp.stream_id == 42
assert resp.http_version == "HTTP/2.0"
assert resp.status_code == 200
- assert resp.msg == ''
+ assert resp.reason == ''
assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
assert resp.content == b''
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index 34bb64f2..3b520a44 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -1,4 +1,5 @@
from netlib.http import cookies
+from netlib.tutils import raises
def test_read_token():
@@ -216,3 +217,18 @@ def test_parse_set_cookie_header():
assert ret2[2].lst == expected[2]
else:
assert ret is None
+
+
+def test_refresh_cookie():
+
+ # Invalid expires format, sent to us by Reddit.
+ c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/"
+ assert cookies.refresh_set_cookie_header(c, 60)
+
+ c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
+ assert "00:21:38" in cookies.refresh_set_cookie_header(c, 60)
+
+ # https://github.com/mitmproxy/mitmproxy/issues/773
+ c = ">=A"
+ with raises(ValueError):
+ cookies.refresh_set_cookie_header(c, 60) \ No newline at end of file
diff --git a/test/netlib/http/test_response.py b/test/netlib/http/test_response.py
index 14588000..a0c44d90 100644
--- a/test/netlib/http/test_response.py
+++ b/test/netlib/http/test_response.py
@@ -1,6 +1,9 @@
from __future__ import absolute_import, print_function, division
+import email
+
import six
+import time
from netlib.http import Headers
from netlib.odict import ODict, ODictCaseless
@@ -100,3 +103,23 @@ class TestResponseUtils(object):
v = resp.cookies
assert len(v) == 1
assert v["foo"] == [["bar", ODictCaseless()]]
+
+ def test_refresh(self):
+ r = tresp()
+ n = time.time()
+ r.headers["date"] = email.utils.formatdate(n)
+ pre = r.headers["date"]
+ r.refresh(n)
+ assert pre == r.headers["date"]
+ r.refresh(n + 60)
+
+ d = email.utils.parsedate_tz(r.headers["date"])
+ d = email.utils.mktime_tz(d)
+ # Weird that this is not exact...
+ assert abs(60 - (d - n)) <= 1
+
+ cookie = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
+ r.headers["set-cookie"] = cookie
+ r.refresh()
+ # Cookie refreshing is tested in test_cookies, we just make sure that it's triggered here.
+ assert cookie != r.headers["set-cookie"]
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-09 17:59:13 +0000