aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--docs/scripting/events.rst8
-rw-r--r--docs/transparent/osx.rst3
-rw-r--r--mitmproxy/addons/core.py3
-rw-r--r--mitmproxy/connections.py18
-rw-r--r--mitmproxy/net/http/cookies.py51
-rw-r--r--mitmproxy/net/tls.py21
-rw-r--r--mitmproxy/proxy/protocol/http_replay.py10
-rw-r--r--mitmproxy/proxy/protocol/tls.py15
-rw-r--r--mitmproxy/tools/console/commander/commander.py2
-rw-r--r--mitmproxy/tools/console/consoleaddons.py13
-rw-r--r--mitmproxy/tools/console/defaultkeys.py6
-rw-r--r--setup.py4
-rw-r--r--test/mitmproxy/net/http/test_cookies.py30
-rw-r--r--test/mitmproxy/net/http/test_response.py2
-rw-r--r--test/mitmproxy/test_connections.py9
-rw-r--r--test/mitmproxy/tools/console/test_keymap.py2
-rw-r--r--web/package.json3
-rw-r--r--web/src/js/components/FlowTable/FlowColumns.jsx2
-rw-r--r--web/src/js/ducks/utils/store.js6
-rw-r--r--web/yarn.lock4
20 files changed, 134 insertions, 78 deletions
diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst
index 4d74b220..d8b1fbb8 100644
--- a/docs/scripting/events.rst
+++ b/docs/scripting/events.rst
@@ -100,10 +100,10 @@ HTTP Events
* - .. py:function:: http_connect(flow)
- Called when we receive an HTTP CONNECT request. Setting a non 2xx
- response on the flow will return the response to the client abort the
- connection. CONNECT requests and responses do not generate the usual
- HTTP handler events. CONNECT requests are only valid in regular and
- upstream proxy modes.
+ response on the flow will return the response to the client and abort
+ the connection. CONNECT requests and responses do not generate the
+ usual HTTP handler events. CONNECT requests are only valid in regular
+ and upstream proxy modes.
*flow*
A ``models.HTTPFlow`` object. The flow is guaranteed to have
diff --git a/docs/transparent/osx.rst b/docs/transparent/osx.rst
index 40e91fac..5d4ec612 100644
--- a/docs/transparent/osx.rst
+++ b/docs/transparent/osx.rst
@@ -17,8 +17,7 @@ Note that this means we don't support transparent mode for earlier versions of O
.. code-block:: none
- rdr on en2 inet proto tcp to any port 80 -> 127.0.0.1 port 8080
- rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080
+ rdr on en0 inet proto tcp to any port {80, 443} -> 127.0.0.1 port 8080
These rules tell pf to redirect all traffic destined for port 80 or 443
to the local mitmproxy instance running on port 8080. You should
diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py
index 2b0b2f14..ca21e1dc 100644
--- a/mitmproxy/addons/core.py
+++ b/mitmproxy/addons/core.py
@@ -164,6 +164,7 @@ class Core:
for f in flows:
p = getattr(f, part, None)
if p:
+ f.backup()
p.decode()
updated.append(f)
ctx.master.addons.trigger("update", updated)
@@ -178,6 +179,7 @@ class Core:
for f in flows:
p = getattr(f, part, None)
if p:
+ f.backup()
current_enc = p.headers.get("content-encoding", "identity")
if current_enc == "identity":
p.encode("deflate")
@@ -204,6 +206,7 @@ class Core:
if p:
current_enc = p.headers.get("content-encoding", "identity")
if current_enc == "identity":
+ f.backup()
p.encode(enc)
updated.append(f)
ctx.master.addons.trigger("update", updated)
diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py
index 86565b7b..9c47985c 100644
--- a/mitmproxy/connections.py
+++ b/mitmproxy/connections.py
@@ -253,7 +253,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
address=address,
ip_address=address,
cert=None,
- sni=None,
+ sni=address[0],
alpn_proto_negotiated=None,
tls_version=None,
source_address=('', 0),
@@ -276,21 +276,21 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
self.wfile.write(message)
self.wfile.flush()
- def establish_tls(self, clientcerts, sni, **kwargs):
+ def establish_tls(self, *, sni=None, client_certs=None, **kwargs):
if sni and not isinstance(sni, str):
raise ValueError("sni must be str, not " + type(sni).__name__)
- clientcert = None
- if clientcerts:
- if os.path.isfile(clientcerts):
- clientcert = clientcerts
+ client_cert = None
+ if client_certs:
+ if os.path.isfile(client_certs):
+ client_cert = client_certs
else:
path = os.path.join(
- clientcerts,
+ client_certs,
self.address[0].encode("idna").decode()) + ".pem"
if os.path.exists(path):
- clientcert = path
+ client_cert = path
- self.convert_to_tls(cert=clientcert, sni=sni, **kwargs)
+ self.convert_to_tls(cert=client_cert, sni=sni, **kwargs)
self.sni = sni
self.alpn_proto_negotiated = self.get_alpn_proto_negotiated()
self.tls_version = self.connection.get_protocol_version_name()
diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py
index 4824bf56..7bef8757 100644
--- a/mitmproxy/net/http/cookies.py
+++ b/mitmproxy/net/http/cookies.py
@@ -114,11 +114,10 @@ def _read_cookie_pairs(s, off=0):
lhs, off = _read_key(s, off)
lhs = lhs.lstrip()
- if lhs:
- rhs = None
- if off < len(s) and s[off] == "=":
- rhs, off = _read_value(s, off + 1, ";")
-
+ rhs = ""
+ if off < len(s) and s[off] == "=":
+ rhs, off = _read_value(s, off + 1, ";")
+ if rhs or lhs:
pairs.append([lhs, rhs])
off += 1
@@ -143,25 +142,24 @@ def _read_set_cookie_pairs(s: str, off=0) -> Tuple[List[TPairs], int]:
lhs, off = _read_key(s, off, ";=,")
lhs = lhs.lstrip()
- if lhs:
- rhs = None
- if off < len(s) and s[off] == "=":
- rhs, off = _read_value(s, off + 1, ";,")
-
- # Special handliing of attributes
- if lhs.lower() == "expires":
- # 'expires' values can contain commas in them so they need to
- # be handled separately.
+ rhs = ""
+ if off < len(s) and s[off] == "=":
+ rhs, off = _read_value(s, off + 1, ";,")
- # We actually bank on the fact that the expires value WILL
- # contain a comma. Things will fail, if they don't.
+ # Special handling of attributes
+ if lhs.lower() == "expires":
+ # 'expires' values can contain commas in them so they need to
+ # be handled separately.
- # '3' is just a heuristic we use to determine whether we've
- # only read a part of the expires value and we should read more.
- if len(rhs) <= 3:
- trail, off = _read_value(s, off + 1, ";,")
- rhs = rhs + "," + trail
+ # We actually bank on the fact that the expires value WILL
+ # contain a comma. Things will fail, if they don't.
+ # '3' is just a heuristic we use to determine whether we've
+ # only read a part of the expires value and we should read more.
+ if len(rhs) <= 3:
+ trail, off = _read_value(s, off + 1, ";,")
+ rhs = rhs + "," + trail
+ if rhs or lhs:
pairs.append([lhs, rhs])
# comma marks the beginning of a new cookie
@@ -196,13 +194,10 @@ def _format_pairs(pairs, specials=(), sep="; "):
"""
vals = []
for k, v in pairs:
- if v is None:
- vals.append(k)
- else:
- if k.lower() not in specials and _has_special(v):
- v = ESCAPE.sub(r"\\\1", v)
- v = '"%s"' % v
- vals.append("%s=%s" % (k, v))
+ if k.lower() not in specials and _has_special(v):
+ v = ESCAPE.sub(r"\\\1", v)
+ v = '"%s"' % v
+ vals.append("%s=%s" % (k, v))
return sep.join(vals)
diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py
index 0e43a2ac..f8eeb44b 100644
--- a/mitmproxy/net/tls.py
+++ b/mitmproxy/net/tls.py
@@ -13,6 +13,7 @@ import certifi
from OpenSSL import SSL
from kaitaistruct import KaitaiStream
+import mitmproxy.options # noqa
from mitmproxy import exceptions, certs
from mitmproxy.contrib.kaitaistruct import tls_client_hello
from mitmproxy.net import check
@@ -57,6 +58,26 @@ METHOD_NAMES = {
}
+def client_arguments_from_options(options: "mitmproxy.options.Options") -> dict:
+
+ if options.ssl_insecure:
+ verify = SSL.VERIFY_NONE
+ else:
+ verify = SSL.VERIFY_PEER
+
+ method, tls_options = VERSION_CHOICES[options.ssl_version_server]
+
+ return {
+ "verify": verify,
+ "method": method,
+ "options": tls_options,
+ "ca_path": options.ssl_verify_upstream_trusted_cadir,
+ "ca_pemfile": options.ssl_verify_upstream_trusted_ca,
+ "client_certs": options.client_certs,
+ "cipher_list": options.ciphers_server,
+ }
+
+
class MasterSecretLogger:
def __init__(self, filename):
self.filename = filename
diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py
index 022e8133..0f3be1ea 100644
--- a/mitmproxy/proxy/protocol/http_replay.py
+++ b/mitmproxy/proxy/protocol/http_replay.py
@@ -9,7 +9,7 @@ from mitmproxy import http
from mitmproxy import flow
from mitmproxy import options
from mitmproxy import connections
-from mitmproxy.net import server_spec
+from mitmproxy.net import server_spec, tls
from mitmproxy.net.http import http1
from mitmproxy.coretypes import basethread
from mitmproxy.utils import human
@@ -76,8 +76,8 @@ class RequestReplayThread(basethread.BaseThread):
if resp.status_code != 200:
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
server.establish_tls(
- self.options.client_certs,
- sni=self.f.server_conn.sni
+ sni=self.f.server_conn.sni,
+ **tls.client_arguments_from_options(self.options)
)
r.first_line_format = "relative"
else:
@@ -91,8 +91,8 @@ class RequestReplayThread(basethread.BaseThread):
server.connect()
if r.scheme == "https":
server.establish_tls(
- self.options.client_certs,
- sni=self.f.server_conn.sni
+ sni=self.f.server_conn.sni,
+ **tls.client_arguments_from_options(self.options)
)
r.first_line_format = "relative"
diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py
index d04c9801..876c1162 100644
--- a/mitmproxy/proxy/protocol/tls.py
+++ b/mitmproxy/proxy/protocol/tls.py
@@ -424,6 +424,9 @@ class TlsLayer(base.Layer):
# * which results in garbage because the layers don' match.
alpn = [self.client_conn.get_alpn_proto_negotiated()]
+ # We pass through the list of ciphers send by the client, because some HTTP/2 servers
+ # will select a non-HTTP/2 compatible cipher from our default list and then hang up
+ # because it's incompatible with h2. :-)
ciphers_server = self.config.options.ciphers_server
if not ciphers_server and self._client_tls:
ciphers_server = []
@@ -432,16 +435,12 @@ class TlsLayer(base.Layer):
ciphers_server.append(CIPHER_ID_NAME_MAP[id])
ciphers_server = ':'.join(ciphers_server)
+ args = net_tls.client_arguments_from_options(self.config.options)
+ args["cipher_list"] = ciphers_server
self.server_conn.establish_tls(
- self.config.client_certs,
- self.server_sni,
- method=self.config.openssl_method_server,
- options=self.config.openssl_options_server,
- verify=self.config.openssl_verification_mode_server,
- ca_path=self.config.options.ssl_verify_upstream_trusted_cadir,
- ca_pemfile=self.config.options.ssl_verify_upstream_trusted_ca,
- cipher_list=ciphers_server,
+ sni=self.server_sni,
alpn_protos=alpn,
+ **args
)
tls_cert_err = self.server_conn.ssl_verification_error
if tls_cert_err is not None:
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index e2088e71..566c42e6 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -47,7 +47,7 @@ CompletionState = typing.NamedTuple(
)
-class CommandBuffer():
+class CommandBuffer:
def __init__(self, master: mitmproxy.master.Master, start: str = "") -> None:
self.master = master
self.text = self.flatten(start)
diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py
index 5907fe95..03f2e240 100644
--- a/mitmproxy/tools/console/consoleaddons.py
+++ b/mitmproxy/tools/console/consoleaddons.py
@@ -277,6 +277,17 @@ class ConsoleAddon:
"""
signals.status_prompt_command.send(partial=" ".join(partial)) # type: ignore
+ @command.command("console.command.set")
+ def console_command_set(self, option: str) -> None:
+ """
+ Prompt the user to set an option of the form "key[=value]".
+ """
+ option_value = getattr(self.master.options, option, None)
+ current_value = option_value if option_value else ""
+ self.master.commands.call(
+ "console.command set %s=%s" % (option, current_value)
+ )
+
@command.command("console.view.keybindings")
def view_keybindings(self) -> None:
"""View the commands list."""
@@ -379,6 +390,8 @@ class ConsoleAddon:
# but for now it is.
if not flow:
raise exceptions.CommandError("No flow selected.")
+ flow.backup()
+
require_dummy_response = (
part in ("response-headers", "response-body", "set-cookies") and
flow.response is None
diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py
index d01d9b7e..084ef262 100644
--- a/mitmproxy/tools/console/defaultkeys.py
+++ b/mitmproxy/tools/console/defaultkeys.py
@@ -26,8 +26,8 @@ def map(km):
km.add("ctrl b", "console.nav.pageup", ["global"], "Page up")
km.add("I", "console.intercept.toggle", ["global"], "Toggle intercept")
- km.add("i", "console.command set intercept=", ["global"], "Set intercept")
- km.add("W", "console.command set save_stream_file=", ["global"], "Stream to file")
+ km.add("i", "console.command.set intercept", ["global"], "Set intercept")
+ km.add("W", "console.command.set save_stream_file", ["global"], "Stream to file")
km.add("A", "flow.resume @all", ["flowlist", "flowview"], "Resume all intercepted flows")
km.add("a", "flow.resume @focus", ["flowlist", "flowview"], "Resume this intercepted flow")
km.add(
@@ -46,7 +46,7 @@ def map(km):
["flowlist", "flowview"],
"Export this flow to file"
)
- km.add("f", "console.command set view_filter=", ["flowlist"], "Set view filter")
+ km.add("f", "console.command.set view_filter", ["flowlist"], "Set view filter")
km.add("F", "set console_focus_follow=toggle", ["flowlist"], "Set focus follow")
km.add(
"ctrl l",
diff --git a/setup.py b/setup.py
index 315c15db..71fe5fd0 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,7 @@ setup(
'h11>=0.7.0,<0.8',
"h2>=3.0.1,<4",
"hyperframe>=5.1.0,<6",
- "kaitaistruct>=0.7, <0.8",
+ "kaitaistruct>=0.7,<0.9",
"ldap3>=2.4,<2.5",
"passlib>=1.6.5, <1.8",
"pyasn1>=0.3.1,<0.5",
@@ -99,7 +99,7 @@ setup(
"rstcheck>=2.2, <4.0",
"sphinx_rtd_theme>=0.1.9, <0.3",
"sphinx-autobuild>=0.5.2, <0.8",
- "sphinx>=1.3.5, <1.7",
+ "sphinx>=1.7,<1.8",
"sphinxcontrib-documentedlist>=0.5.0, <0.7",
"tox>=2.3, <3",
],
diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py
index 77549d9e..e12b0f00 100644
--- a/test/mitmproxy/net/http/test_cookies.py
+++ b/test/mitmproxy/net/http/test_cookies.py
@@ -7,6 +7,10 @@ from mitmproxy.net.http import cookies
cookie_pairs = [
[
+ "=uno",
+ [["", "uno"]]
+ ],
+ [
"",
[]
],
@@ -16,7 +20,7 @@ cookie_pairs = [
],
[
"one",
- [["one", None]]
+ [["one", ""]]
],
[
"one=uno; two=due",
@@ -36,7 +40,7 @@ cookie_pairs = [
],
[
"one=uno; two; three=tre",
- [["one", "uno"], ["two", None], ["three", "tre"]]
+ [["one", "uno"], ["two", ""], ["three", "tre"]]
],
[
"_lvs2=zHai1+Hq+Tc2vmc2r4GAbdOI5Jopg3EwsdUT9g=; "
@@ -79,8 +83,12 @@ def test_read_quoted_string():
def test_read_cookie_pairs():
vals = [
[
+ "=uno",
+ [["", "uno"]]
+ ],
+ [
"one",
- [["one", None]]
+ [["one", ""]]
],
[
"one=two",
@@ -100,7 +108,7 @@ def test_read_cookie_pairs():
],
[
'one="two"; three=four; five',
- [["one", "two"], ["three", "four"], ["five", None]]
+ [["one", "two"], ["three", "four"], ["five", ""]]
],
[
'one="\\"two"; three=four',
@@ -135,6 +143,12 @@ def test_cookie_roundtrips():
def test_parse_set_cookie_pairs():
pairs = [
[
+ "=uno",
+ [[
+ ["", "uno"]
+ ]]
+ ],
+ [
"one=uno",
[[
["one", "uno"]
@@ -150,7 +164,7 @@ def test_parse_set_cookie_pairs():
"one=uno; foo",
[[
["one", "uno"],
- ["foo", None]
+ ["foo", ""]
]]
],
[
@@ -200,6 +214,12 @@ def test_parse_set_cookie_header():
";", []
],
[
+ "=uno",
+ [
+ ("", "uno", ())
+ ]
+ ],
+ [
"one=uno",
[
("one", "uno", ())
diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py
index af35bab3..f3470384 100644
--- a/test/mitmproxy/net/http/test_response.py
+++ b/test/mitmproxy/net/http/test_response.py
@@ -113,7 +113,7 @@ class TestResponseUtils:
assert attrs["domain"] == "example.com"
assert attrs["expires"] == "Wed Oct 21 16:29:41 2015"
assert attrs["path"] == "/"
- assert attrs["httponly"] is None
+ assert attrs["httponly"] == ""
def test_get_cookies_no_value(self):
resp = tresp()
diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py
index 9e5d89f1..00cdbc87 100644
--- a/test/mitmproxy/test_connections.py
+++ b/test/mitmproxy/test_connections.py
@@ -155,7 +155,7 @@ class TestServerConnection:
def test_sni(self):
c = connections.ServerConnection(('', 1234))
with pytest.raises(ValueError, matches='sni must be str, not '):
- c.establish_tls(None, b'foobar')
+ c.establish_tls(sni=b'foobar')
def test_state(self):
c = tflow.tserver_conn()
@@ -222,17 +222,16 @@ class TestServerConnectionTLS(tservers.ServerTestBase):
def handle(self):
self.finish()
- @pytest.mark.parametrize("clientcert", [
+ @pytest.mark.parametrize("client_certs", [
None,
tutils.test_data.path("mitmproxy/data/clientcert"),
tutils.test_data.path("mitmproxy/data/clientcert/client.pem"),
])
- def test_tls(self, clientcert):
+ def test_tls(self, client_certs):
c = connections.ServerConnection(("127.0.0.1", self.port))
c.connect()
- c.establish_tls(clientcert, "foo.com")
+ c.establish_tls(client_certs=client_certs)
assert c.connected()
- assert c.sni == "foo.com"
assert c.tls_established
c.close()
c.finish()
diff --git a/test/mitmproxy/tools/console/test_keymap.py b/test/mitmproxy/tools/console/test_keymap.py
index 00e64991..7b475ff8 100644
--- a/test/mitmproxy/tools/console/test_keymap.py
+++ b/test/mitmproxy/tools/console/test_keymap.py
@@ -42,7 +42,7 @@ def test_join():
km = keymap.Keymap(tctx.master)
km.add("key", "str", ["options"], "help1")
km.add("key", "str", ["commands"])
- return
+
assert len(km.bindings) == 1
assert len(km.bindings[0].contexts) == 2
assert km.bindings[0].help == "help1"
diff --git a/web/package.json b/web/package.json
index 31c2d6d6..77b13e8b 100644
--- a/web/package.json
+++ b/web/package.json
@@ -37,7 +37,8 @@
"redux-logger": "^3.0.6",
"redux-mock-store": "^1.3.0",
"redux-thunk": "^2.2.0",
- "shallowequal": "^1.0.2"
+ "shallowequal": "^1.0.2",
+ "stable": "^0.1.6"
},
"devDependencies": {
"babel-core": "^6.26.0",
diff --git a/web/src/js/components/FlowTable/FlowColumns.jsx b/web/src/js/components/FlowTable/FlowColumns.jsx
index 02a4fba1..e60ed487 100644
--- a/web/src/js/components/FlowTable/FlowColumns.jsx
+++ b/web/src/js/components/FlowTable/FlowColumns.jsx
@@ -119,7 +119,7 @@ export function TimeColumn({ flow }) {
return (
<td className="col-time">
{flow.response ? (
- formatTimeDelta(1000 * (flow.response.timestamp_end - flow.server_conn.timestamp_start))
+ formatTimeDelta(1000 * (flow.response.timestamp_end - flow.request.timestamp_start))
) : (
'...'
)}
diff --git a/web/src/js/ducks/utils/store.js b/web/src/js/ducks/utils/store.js
index ac272650..ad2242ee 100644
--- a/web/src/js/ducks/utils/store.js
+++ b/web/src/js/ducks/utils/store.js
@@ -1,3 +1,5 @@
+import stable from 'stable'
+
export const SET_FILTER = 'LIST_SET_FILTER'
export const SET_SORT = 'LIST_SET_SORT'
export const ADD = 'LIST_ADD'
@@ -35,7 +37,7 @@ export default function reduce(state = defaultState, action) {
switch (action.type) {
case SET_FILTER:
- view = list.filter(action.filter).sort(action.sort)
+ view = stable(list.filter(action.filter), action.sort)
viewIndex = {}
view.forEach((item, index) => {
viewIndex[item.id] = index
@@ -43,7 +45,7 @@ export default function reduce(state = defaultState, action) {
break
case SET_SORT:
- view = [...view].sort(action.sort)
+ view = stable([...view], action.sort)
viewIndex = {}
view.forEach((item, index) => {
viewIndex[item.id] = index
diff --git a/web/yarn.lock b/web/yarn.lock
index aa5ae85f..1930fded 100644
--- a/web/yarn.lock
+++ b/web/yarn.lock
@@ -5449,6 +5449,10 @@ sshpk@^1.7.0:
jsbn "~0.1.0"
tweetnacl "~0.14.0"
+stable@^0.1.6:
+ version "0.1.6"
+ resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.6.tgz#910f5d2aed7b520c6e777499c1f32e139fdecb10"
+
statuses@1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e"
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-26 01:46:59 +0000