diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | mitmproxy/contentviews/xml_html.py | 2 | ||||
| -rw-r--r-- | mitmproxy/io/__init__.py | 3 | ||||
| -rw-r--r-- | mitmproxy/io/db.py | 40 | ||||
| -rw-r--r-- | mitmproxy/io/proto/http.proto | 93 | ||||
| -rw-r--r-- | mitmproxy/io/proto/http_pb2.py | 779 | ||||
| -rw-r--r-- | mitmproxy/io/protobuf.py | 198 | ||||
| -rw-r--r-- | mitmproxy/tools/console/commander/commander.py | 49 | ||||
| -rw-r--r-- | mitmproxy/tools/console/statusbar.py | 7 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | setup.py | 1 | ||||
| -rw-r--r-- | test/bench/serialization-bm.py | 116 | ||||
| -rwxr-xr-x | test/filename_matching.py | 3 | ||||
| -rw-r--r-- | test/mitmproxy/io/test_db.py | 26 | ||||
| -rw-r--r-- | test/mitmproxy/io/test_protobuf.py | 120 | ||||
| -rw-r--r-- | test/mitmproxy/tools/console/test_commander.py | 62 |
16 files changed, 1496 insertions, 6 deletions
@@ -5,6 +5,7 @@ MANIFEST *.py[cdo] *.swp *.swo +*.sqlite *.egg-info/ .coverage* .idea diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 22c402e6..658fbcd7 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -138,7 +138,7 @@ def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) if is_inline_text(prev2, prev1, t) or is_inline_text(t, next1, next2): return True if isinstance(next1, Tag) and t.is_opening and next1.is_closing and t.tag == next1.tag: - return True # <div></div> (start tag) + return True # <div></div> (start tag) if isinstance(prev1, Tag) and prev1.is_opening and t.is_closing and prev1.tag == t.tag: return True # <div></div> (end tag) return False diff --git a/mitmproxy/io/__init__.py b/mitmproxy/io/__init__.py index 540e6871..bd248fd2 100644 --- a/mitmproxy/io/__init__.py +++ b/mitmproxy/io/__init__.py @@ -1,7 +1,8 @@ from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths +from .db import DBHandler __all__ = [ - "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths" + "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths", "DBHandler" ] diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py new file mode 100644 index 00000000..ea42e08b --- /dev/null +++ b/mitmproxy/io/db.py @@ -0,0 +1,40 @@ +import sqlite3 +import os + +from mitmproxy.io import protobuf + + +class DBHandler: + + """ + This class is wrapping up connection to SQLITE DB. + """ + + def __init__(self, db_path, mode='load'): + if mode == 'write': + if os.path.isfile(db_path): + os.remove(db_path) + self.db_path = db_path + self._con = sqlite3.connect(self.db_path) + self._c = self._con.cursor() + self._create_db() + + def _create_db(self): + with self._con: + self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS(' + 'id INTEGER PRIMARY KEY,' + 'pbuf_blob BLOB)') + + def store(self, flows): + blobs = [] + for flow in flows: + blobs.append((protobuf.dumps(flow),)) + with self._con: + self._con.executemany('INSERT INTO FLOWS (pbuf_blob) values (?)', blobs) + + def load(self): + flows = [] + self._c.execute('SELECT pbuf_blob FROM FLOWS') + for row in self._c.fetchall(): + flows.append((protobuf.loads(row[0]))) + return flows diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto new file mode 100644 index 00000000..c86a04f3 --- /dev/null +++ b/mitmproxy/io/proto/http.proto @@ -0,0 +1,93 @@ +syntax='proto2'; + +message HTTPFlow { + optional HTTPRequest request = 1; + optional HTTPResponse response = 2; + optional HTTPError error = 3; + optional ClientConnection client_conn = 4; + optional ServerConnection server_conn = 5; + optional bool intercepted = 6; + optional bool marked = 7; + optional string mode = 8; + optional string id = 9; +} + +message HTTPRequest { + optional string first_line_format = 1; + optional string method = 2; + optional string scheme = 3; + optional string host = 4; + optional int32 port = 5; + optional string path = 6; + optional string http_version = 7; + repeated HTTPHeader headers = 8; + optional bytes content = 9; + optional double timestamp_start = 10; + optional double timestamp_end = 11; + optional bool is_replay = 12; +} + +message HTTPResponse { + optional string http_version = 1; + optional int32 status_code = 2; + optional string reason = 3; + repeated HTTPHeader headers = 4; + optional bytes content = 5; + optional double timestamp_start = 6; + optional double timestamp_end = 7; + optional bool is_replay = 8; +} + +message HTTPError { + optional string msg = 1; + optional double timestamp = 2; +} + +message HTTPHeader { + optional string name = 1; + optional string value = 2; +} + + +message Address { + optional string host = 1; + optional int32 port = 2; +} + +message ClientConnection { + optional string id = 1; + optional Address address = 2; + optional bool tls_established = 3; + optional string clientcert = 4; + optional string mitmcert = 5; + optional double timestamp_start = 6; + optional double timestamp_tls_setup = 7; + optional double timestamp_end = 8; + optional string sni = 9; + optional string cipher_name = 10; + optional bytes alpn_proto_negotiated = 11; + optional string tls_version = 12; + repeated TLSExtension tls_extensions = 13; +} + +message ServerConnection { + optional string id = 1; + optional Address address = 2; + optional Address ip_address = 3; + optional Address source_address = 4; + optional bool tls_established = 5; + optional string cert = 6; + optional string sni = 7; + optional bytes alpn_proto_negotiated = 8; + optional string tls_version = 9; + optional double timestamp_start = 10; + optional double timestamp_tcp_setup = 11; + optional double timestamp_tls_setup = 12; + optional double timestamp_end = 13; + optional ServerConnection via = 14; +} + +message TLSExtension { + optional int64 int = 1; + optional bytes bytes = 2; +}
\ No newline at end of file diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py new file mode 100644 index 00000000..47b09815 --- /dev/null +++ b/mitmproxy/io/proto/http_pb2.py @@ -0,0 +1,779 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: http.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='http.proto', + package='', + syntax='proto2', + serialized_pb=_b('\n\nhttp.proto\"\xf4\x01\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x01\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x01\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x01\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x01\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x01\x12\x15\n\rtimestamp_end\x18\r \x01(\x01\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') +) + + + + +_HTTPFLOW = _descriptor.Descriptor( + name='HTTPFlow', + full_name='HTTPFlow', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='request', full_name='HTTPFlow.request', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='response', full_name='HTTPFlow.response', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='error', full_name='HTTPFlow.error', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='client_conn', full_name='HTTPFlow.client_conn', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='server_conn', full_name='HTTPFlow.server_conn', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='intercepted', full_name='HTTPFlow.intercepted', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='marked', full_name='HTTPFlow.marked', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mode', full_name='HTTPFlow.mode', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='id', full_name='HTTPFlow.id', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=15, + serialized_end=259, +) + + +_HTTPREQUEST = _descriptor.Descriptor( + name='HTTPRequest', + full_name='HTTPRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='first_line_format', full_name='HTTPRequest.first_line_format', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='method', full_name='HTTPRequest.method', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='scheme', full_name='HTTPRequest.scheme', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='host', full_name='HTTPRequest.host', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='HTTPRequest.port', index=4, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='path', full_name='HTTPRequest.path', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPRequest.http_version', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPRequest.headers', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPRequest.content', index=8, + number=9, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9, + number=10, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10, + number=11, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPRequest.is_replay', index=11, + number=12, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=262, + serialized_end=512, +) + + +_HTTPRESPONSE = _descriptor.Descriptor( + name='HTTPResponse', + full_name='HTTPResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPResponse.http_version', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='status_code', full_name='HTTPResponse.status_code', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reason', full_name='HTTPResponse.reason', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPResponse.headers', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPResponse.content', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6, + number=7, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPResponse.is_replay', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=515, + serialized_end=702, +) + + +_HTTPERROR = _descriptor.Descriptor( + name='HTTPError', + full_name='HTTPError', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='msg', full_name='HTTPError.msg', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp', full_name='HTTPError.timestamp', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=704, + serialized_end=747, +) + + +_HTTPHEADER = _descriptor.Descriptor( + name='HTTPHeader', + full_name='HTTPHeader', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='HTTPHeader.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='HTTPHeader.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=749, + serialized_end=790, +) + + +_ADDRESS = _descriptor.Descriptor( + name='Address', + full_name='Address', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='host', full_name='Address.host', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='Address.port', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=792, + serialized_end=829, +) + + +_CLIENTCONNECTION = _descriptor.Descriptor( + name='ClientConnection', + full_name='ClientConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ClientConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ClientConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ClientConnection.tls_established', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='clientcert', full_name='ClientConnection.clientcert', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mitmcert', full_name='ClientConnection.mitmcert', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6, + number=7, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7, + number=8, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ClientConnection.sni', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cipher_name', full_name='ClientConnection.cipher_name', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ClientConnection.alpn_proto_negotiated', index=10, + number=11, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ClientConnection.tls_version', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_extensions', full_name='ClientConnection.tls_extensions', index=12, + number=13, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=832, + serialized_end=1154, +) + + +_SERVERCONNECTION = _descriptor.Descriptor( + name='ServerConnection', + full_name='ServerConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ServerConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ServerConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='ip_address', full_name='ServerConnection.ip_address', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='source_address', full_name='ServerConnection.source_address', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ServerConnection.tls_established', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cert', full_name='ServerConnection.cert', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ServerConnection.sni', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ServerConnection.alpn_proto_negotiated', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ServerConnection.tls_version', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9, + number=10, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10, + number=11, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11, + number=12, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12, + number=13, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='via', full_name='ServerConnection.via', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1157, + serialized_end=1520, +) + + +_TLSEXTENSION = _descriptor.Descriptor( + name='TLSExtension', + full_name='TLSExtension', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='int', full_name='TLSExtension.int', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bytes', full_name='TLSExtension.bytes', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1522, + serialized_end=1564, +) + +_HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST +_HTTPFLOW.fields_by_name['response'].message_type = _HTTPRESPONSE +_HTTPFLOW.fields_by_name['error'].message_type = _HTTPERROR +_HTTPFLOW.fields_by_name['client_conn'].message_type = _CLIENTCONNECTION +_HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION +_HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER +_HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER +_CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_CLIENTCONNECTION.fields_by_name['tls_extensions'].message_type = _TLSEXTENSION +_SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['via'].message_type = _SERVERCONNECTION +DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW +DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST +DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE +DESCRIPTOR.message_types_by_name['HTTPError'] = _HTTPERROR +DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER +DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS +DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION +DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION +DESCRIPTOR.message_types_by_name['TLSExtension'] = _TLSEXTENSION +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict( + DESCRIPTOR = _HTTPFLOW, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPFlow) + )) +_sym_db.RegisterMessage(HTTPFlow) + +HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), dict( + DESCRIPTOR = _HTTPREQUEST, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPRequest) + )) +_sym_db.RegisterMessage(HTTPRequest) + +HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), dict( + DESCRIPTOR = _HTTPRESPONSE, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPResponse) + )) +_sym_db.RegisterMessage(HTTPResponse) + +HTTPError = _reflection.GeneratedProtocolMessageType('HTTPError', (_message.Message,), dict( + DESCRIPTOR = _HTTPERROR, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPError) + )) +_sym_db.RegisterMessage(HTTPError) + +HTTPHeader = _reflection.GeneratedProtocolMessageType('HTTPHeader', (_message.Message,), dict( + DESCRIPTOR = _HTTPHEADER, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPHeader) + )) +_sym_db.RegisterMessage(HTTPHeader) + +Address = _reflection.GeneratedProtocolMessageType('Address', (_message.Message,), dict( + DESCRIPTOR = _ADDRESS, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:Address) + )) +_sym_db.RegisterMessage(Address) + +ClientConnection = _reflection.GeneratedProtocolMessageType('ClientConnection', (_message.Message,), dict( + DESCRIPTOR = _CLIENTCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ClientConnection) + )) +_sym_db.RegisterMessage(ClientConnection) + +ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', (_message.Message,), dict( + DESCRIPTOR = _SERVERCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ServerConnection) + )) +_sym_db.RegisterMessage(ServerConnection) + +TLSExtension = _reflection.GeneratedProtocolMessageType('TLSExtension', (_message.Message,), dict( + DESCRIPTOR = _TLSEXTENSION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:TLSExtension) + )) +_sym_db.RegisterMessage(TLSExtension) + + +# @@protoc_insertion_point(module_scope) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py new file mode 100644 index 00000000..9a00eacf --- /dev/null +++ b/mitmproxy/io/protobuf.py @@ -0,0 +1,198 @@ +import typing + +from mitmproxy import flow +from mitmproxy import exceptions +from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest +from mitmproxy.certs import Cert +from mitmproxy.connections import ClientConnection, ServerConnection +from mitmproxy.io.proto import http_pb2 + + +def _move_attrs(s_obj, d_obj, attrs): + for attr in attrs: + if not isinstance(d_obj, dict): + if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: + setattr(d_obj, attr, getattr(s_obj, attr)) + else: + if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: + # ugly fix to set None in empty str or bytes fields + if getattr(s_obj, attr) == "" or getattr(s_obj, attr) == b"": + d_obj[attr] = None + else: + d_obj[attr] = getattr(s_obj, attr) + + +def _dump_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: + pres = http_pb2.HTTPResponse() + _move_attrs(res, pres, ['http_version', 'status_code', 'reason', + 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + if res.headers: + for h in res.headers.fields: + header = pres.headers.add() + header.name = h[0] + header.value = h[1] + return pres + + +def _dump_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: + preq = http_pb2.HTTPRequest() + _move_attrs(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', + 'timestamp_start', 'timestamp_end', 'is_replay']) + if req.headers: + for h in req.headers.fields: + header = preq.headers.add() + header.name = h[0] + header.value = h[1] + return preq + + +def _dump_http_client_conn(cc: ClientConnection) -> http_pb2.ClientConnection: + pcc = http_pb2.ClientConnection() + _move_attrs(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni', + 'cipher_name', 'alpn_proto_negotiated', 'tls_version']) + for cert in ['clientcert', 'mitmcert']: + if hasattr(cc, cert) and getattr(cc, cert) is not None: + setattr(pcc, cert, getattr(cc, cert).to_pem()) + if cc.tls_extensions: + for extension in cc.tls_extensions: + ext = pcc.tls_extensions.add() + ext.int = extension[0] + ext.bytes = extension[1] + if cc.address: + pcc.address.host = cc.address[0] + pcc.address.port = cc.address[1] + return pcc + + +def _dump_http_server_conn(sc: ServerConnection) -> http_pb2.ServerConnection: + psc = http_pb2.ServerConnection() + _move_attrs(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for addr in ['address', 'ip_address', 'source_address']: + if hasattr(sc, addr) and getattr(sc, addr) is not None: + getattr(psc, addr).host = getattr(sc, addr)[0] + getattr(psc, addr).port = getattr(sc, addr)[1] + if sc.cert: + psc.cert = sc.cert.to_pem() + if sc.via: + psc.via.MergeFrom(_dump_http_server_conn(sc.via)) + return psc + + +def _dump_http_error(e: flow.Error) -> http_pb2.HTTPError: + pe = http_pb2.HTTPError() + for attr in ['msg', 'timestamp']: + if hasattr(e, attr) and getattr(e, attr) is not None: + setattr(pe, attr, getattr(e, attr)) + return pe + + +def dump_http(f: flow.Flow) -> http_pb2.HTTPFlow: + pf = http_pb2.HTTPFlow() + for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: + if hasattr(f, p) and getattr(f, p): + getattr(pf, p).MergeFrom(eval(f"_dump_http_{p}")(getattr(f, p))) + _move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id']) + return pf + + +def dumps(f: flow.Flow) -> bytes: + if f.type != "http": + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + p = dump_http(f) + return p.SerializeToString() + + +def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: + d: dict = {} + _move_attrs(o, d, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', + 'timestamp_start', 'timestamp_end', 'is_replay']) + if d['content'] is None: + d['content'] = b"" + d["headers"] = [] + for header in o.headers: + d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) + + return HTTPRequest(**d) + + +def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: + d: dict = {} + _move_attrs(o, d, ['http_version', 'status_code', 'reason', + 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + if d['content'] is None: + d['content'] = b"" + d["headers"] = [] + for header in o.headers: + d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) + + return HTTPResponse(**d) + + +def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: + d: dict = {} + _move_attrs(o, d, ['id', 'tls_established', 'sni', 'cipher_name', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for cert in ['clientcert', 'mitmcert']: + if hasattr(o, cert) and getattr(o, cert): + d[cert] = Cert.from_pem(getattr(o, cert)) + if o.tls_extensions: + d['tls_extensions'] = [] + for extension in o.tls_extensions: + d['tls_extensions'].append((extension.int, extension.bytes)) + if o.address: + d['address'] = (o.address.host, o.address.port) + cc = ClientConnection(None, tuple(), None) + for k, v in d.items(): + setattr(cc, k, v) + return cc + + +def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection: + d: dict = {} + _move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for addr in ['address', 'ip_address', 'source_address']: + if hasattr(o, addr): + d[addr] = (getattr(o, addr).host, getattr(o, addr).port) + if o.cert: + c = Cert.from_pem(o.cert) + d['cert'] = c + if o.HasField('via'): + d['via'] = _load_http_server_conn(o.via) + sc = ServerConnection(tuple()) + for k, v in d.items(): + setattr(sc, k, v) + return sc + + +def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]: + d = {} + for m in ['msg', 'timestamp']: + if hasattr(o, m) and getattr(o, m): + d[m] = getattr(o, m) + return None if not d else flow.Error(**d) + + +def load_http(hf: http_pb2.HTTPFlow) -> HTTPFlow: + parts = {} + for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: + if hf.HasField(p): + parts[p] = eval(f"_load_http_{p}")(getattr(hf, p)) + else: + parts[p] = None + _move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id']) + f = HTTPFlow(ClientConnection(None, tuple(), None), ServerConnection(tuple())) + for k, v in parts.items(): + setattr(f, k, v) + return f + + +def loads(b: bytes, typ="http") -> flow.Flow: + if typ != 'http': + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + p = http_pb2.HTTPFlow() + p.ParseFromString(b) + return load_http(p) diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index df3eaa5a..e8550f86 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -1,5 +1,7 @@ import abc +import copy import typing +import collections import urwid from urwid.text_layout import calc_coords @@ -156,13 +158,53 @@ class CommandBuffer: self.completion = None +class CommandHistory: + def __init__(self, master: mitmproxy.master.Master, size: int=30) -> None: + self.saved_commands: collections.deque = collections.deque( + [CommandBuffer(master, "")], + maxlen=size + ) + self.index: int = 0 + + @property + def last_index(self): + return len(self.saved_commands) - 1 + + def get_next(self) -> typing.Optional[CommandBuffer]: + if self.index < self.last_index: + self.index = self.index + 1 + return self.saved_commands[self.index] + return None + + def get_prev(self) -> typing.Optional[CommandBuffer]: + if self.index > 0: + self.index = self.index - 1 + return self.saved_commands[self.index] + return None + + def add_command(self, command: CommandBuffer, execution: bool=False) -> None: + if self.index == self.last_index or execution: + last_item = self.saved_commands[-1] + last_item_empty = not last_item.text + if last_item.text == command.text or (last_item_empty and execution): + self.saved_commands[-1] = copy.copy(command) + else: + self.saved_commands.append(command) + if not execution and self.index < self.last_index: + self.index += 1 + if execution: + self.index = self.last_index + + class CommandEdit(urwid.WidgetWrap): leader = ": " - def __init__(self, master: mitmproxy.master.Master, text: str) -> None: + def __init__(self, master: mitmproxy.master.Master, + text: str, history: CommandHistory) -> None: super().__init__(urwid.Text(self.leader)) self.master = master self.cbuf = CommandBuffer(master, text) + self.history = history self.update() def keypress(self, size, key): @@ -172,6 +214,11 @@ class CommandEdit(urwid.WidgetWrap): self.cbuf.left() elif key == "right": self.cbuf.right() + elif key == "up": + self.history.add_command(self.cbuf) + self.cbuf = self.history.get_prev() or self.cbuf + elif key == "down": + self.cbuf = self.history.get_next() or self.cbuf elif key == "tab": self.cbuf.cycle_completion() elif len(key) == 1: diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 215cf500..2d32f487 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -42,6 +42,8 @@ class ActionBar(urwid.WidgetWrap): signals.status_prompt_onekey.connect(self.sig_prompt_onekey) signals.status_prompt_command.connect(self.sig_prompt_command) + self.command_history = commander.CommandHistory(master) + self.prompting = None self.onekey = False @@ -98,7 +100,8 @@ class ActionBar(urwid.WidgetWrap): def sig_prompt_command(self, sender, partial=""): signals.focus.send(self, section="footer") - self._w = commander.CommandEdit(self.master, partial) + self._w = commander.CommandEdit(self.master, partial, + self.command_history) self.prompting = commandexecutor.CommandExecutor(self.master) def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()): @@ -125,6 +128,7 @@ class ActionBar(urwid.WidgetWrap): def keypress(self, size, k): if self.prompting: if k == "esc": + self.command_history.index = self.command_history.last_index self.prompt_done() elif self.onekey: if k == "enter": @@ -132,6 +136,7 @@ class ActionBar(urwid.WidgetWrap): elif k in self.onekey: self.prompt_execute(k) elif k == "enter": + self.command_history.add_command(self._w.cbuf, True) self.prompt_execute(self._w.get_edit_text()) else: if common.is_keypress(k): @@ -2,7 +2,7 @@ max-line-length = 140 max-complexity = 25 ignore = E251,C901,W503,W292,E722,E741 -exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* +exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp [tool:pytest] @@ -70,6 +70,7 @@ setup( "kaitaistruct>=0.7,<0.9", "ldap3>=2.5,<2.6", "passlib>=1.6.5, <1.8", + "protobuf>=3.6.0, <3.7", "pyasn1>=0.3.1,<0.5", "pyOpenSSL>=17.5,<18.1", "pyparsing>=2.1.3, <2.3", diff --git a/test/bench/serialization-bm.py b/test/bench/serialization-bm.py new file mode 100644 index 00000000..665b72cb --- /dev/null +++ b/test/bench/serialization-bm.py @@ -0,0 +1,116 @@ +import tempfile
+import asyncio
+import typing
+import time
+
+from statistics import mean
+
+from mitmproxy import ctx
+from mitmproxy.io import db
+from mitmproxy.test import tflow
+
+
+class StreamTester:
+
+ """
+ Generates a constant stream of flows and
+ measure protobuf dumping throughput.
+ """
+
+ def __init__(self):
+ self.dbh = None
+ self.streaming = False
+ self.tf = None
+ self.out = None
+ self.hot_flows = []
+ self.results = []
+ self._flushes = 0
+ self._stream_period = 0.001
+ self._flush_period = 3.0
+ self._flush_rate = 150
+ self._target = 2000
+ self.loop = asyncio.get_event_loop()
+ self.queue = asyncio.Queue(maxsize=self._flush_rate * 3, loop=self.loop)
+ self.temp = tempfile.NamedTemporaryFile()
+
+ def load(self, loader):
+ loader.add_option(
+ "testflow_size",
+ int,
+ 1000,
+ "Length in bytes of test flow content"
+ )
+ loader.add_option(
+ "benchmark_save_path",
+ typing.Optional[str],
+ None,
+ "Destination for the stats result file"
+ )
+
+ def _log(self, msg):
+ if self.out:
+ self.out.write(msg + '\n')
+ else:
+ ctx.log(msg)
+
+ def running(self):
+ if not self.streaming:
+ ctx.log("<== Serialization Benchmark Enabled ==>")
+ self.tf = tflow.tflow()
+ self.tf.request.content = b'A' * ctx.options.testflow_size
+ ctx.log(f"With content size: {len(self.tf.request.content)} B")
+ if ctx.options.benchmark_save_path:
+ ctx.log(f"Storing results to {ctx.options.benchmark_save_path}")
+ self.out = open(ctx.options.benchmark_save_path, "w")
+ self.dbh = db.DBHandler(self.temp.name, mode='write')
+ self.streaming = True
+ tasks = (self.stream, self.writer, self.stats)
+ self.loop.create_task(asyncio.gather(*(t() for t in tasks)))
+
+ async def stream(self):
+ while True:
+ await self.queue.put(self.tf)
+ await asyncio.sleep(self._stream_period)
+
+ async def writer(self):
+ while True:
+ await asyncio.sleep(self._flush_period)
+ count = 1
+ f = await self.queue.get()
+ self.hot_flows.append(f)
+ while count < self._flush_rate:
+ try:
+ self.hot_flows.append(self.queue.get_nowait())
+ count += 1
+ except asyncio.QueueEmpty:
+ pass
+ start = time.perf_counter()
+ n = self._fflush()
+ end = time.perf_counter()
+ self._log(f"dumps/time ratio: {n} / {end-start} -> {n/(end-start)}")
+ self.results.append(n / (end - start))
+ self._flushes += n
+ self._log(f"Flows dumped: {self._flushes}")
+ ctx.log(f"Progress: {min(100.0, 100.0 * (self._flushes / self._target))}%")
+
+ async def stats(self):
+ while True:
+ await asyncio.sleep(1.0)
+ if self._flushes >= self._target:
+ self._log(f"AVG : {mean(self.results)}")
+ ctx.log(f"<== Benchmark Ended. Shutting down... ==>")
+ if self.out:
+ self.out.close()
+ self.temp.close()
+ ctx.master.shutdown()
+
+ def _fflush(self):
+ self.dbh.store(self.hot_flows)
+ n = len(self.hot_flows)
+ self.hot_flows = []
+ return n
+
+
+addons = [
+ StreamTester()
+]
diff --git a/test/filename_matching.py b/test/filename_matching.py index 5f49725e..f5321307 100755 --- a/test/filename_matching.py +++ b/test/filename_matching.py @@ -9,7 +9,8 @@ import sys def check_src_files_have_test(): missing_test_files = [] - excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] + excluded = ['mitmproxy/contrib/', 'mitmproxy/io/proto/', + 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] diff --git a/test/mitmproxy/io/test_db.py b/test/mitmproxy/io/test_db.py new file mode 100644 index 00000000..4a2dfb67 --- /dev/null +++ b/test/mitmproxy/io/test_db.py @@ -0,0 +1,26 @@ +from mitmproxy.io import db +from mitmproxy.test import tflow + + +class TestDB: + + def test_create(self, tdata): + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite") + with dh._con as c: + cur = c.cursor() + cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='FLOWS';") + assert cur.fetchall() == [('FLOWS',)] + + def test_roundtrip(self, tdata): + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite", mode='write') + flows = [] + for i in range(10): + flows.append(tflow.tflow()) + dh.store(flows) + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite") + with dh._con as c: + cur = c.cursor() + cur.execute("SELECT count(*) FROM FLOWS;") + assert cur.fetchall()[0][0] == 10 + loaded_flows = dh.load() + assert len(loaded_flows) == len(flows) diff --git a/test/mitmproxy/io/test_protobuf.py b/test/mitmproxy/io/test_protobuf.py new file mode 100644 index 00000000..f725b980 --- /dev/null +++ b/test/mitmproxy/io/test_protobuf.py @@ -0,0 +1,120 @@ +import pytest + +from mitmproxy import certs +from mitmproxy import http +from mitmproxy import exceptions +from mitmproxy.test import tflow, tutils +from mitmproxy.io import protobuf + + +class TestProtobuf: + + def test_roundtrip_client(self): + c = tflow.tclient_conn() + del c.reply + c.rfile = None + c.wfile = None + pc = protobuf._dump_http_client_conn(c) + lc = protobuf._load_http_client_conn(pc) + assert c.__dict__ == lc.__dict__ + + def test_roundtrip_client_cert(self, tdata): + c = tflow.tclient_conn() + c.rfile = None + c.wfile = None + del c.reply + with open(tdata.path("mitmproxy/net/data/clientcert/client.pem"), "rb") as f: + d = f.read() + c.clientcert = certs.Cert.from_pem(d) + pc = protobuf._dump_http_client_conn(c) + lc = protobuf._load_http_client_conn(pc) + assert c.__dict__ == lc.__dict__ + + def test_roundtrip_server(self): + s = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + + def test_roundtrip_server_cert(self, tdata): + s = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f: + d = f.read() + s.cert = certs.Cert.from_pem(d) + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + + def test_roundtrip_server_via(self): + s = tflow.tserver_conn() + s.via = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + del s.via.reply + s.via.wfile = None + s.via.rfile = None + assert s.via.__dict__ == ls.via.__dict__ + + def test_roundtrip_http_request(self): + req = http.HTTPRequest.wrap(tutils.treq()) + preq = protobuf._dump_http_request(req) + lreq = protobuf._load_http_request(preq) + assert req.__dict__ == lreq.__dict__ + + def test_roundtrip_http_request_empty_content(self): + req = http.HTTPRequest.wrap(tutils.treq(content=b"")) + preq = protobuf._dump_http_request(req) + lreq = protobuf._load_http_request(preq) + assert req.__dict__ == lreq.__dict__ + + def test_roundtrip_http_response(self): + res = http.HTTPResponse.wrap(tutils.tresp()) + pres = protobuf._dump_http_response(res) + lres = protobuf._load_http_response(pres) + assert res.__dict__ == lres.__dict__ + + def test_roundtrip_http_response_empty_content(self): + res = http.HTTPResponse.wrap(tutils.tresp(content=b"")) + pres = protobuf._dump_http_response(res) + lres = protobuf._load_http_response(pres) + assert res.__dict__ == lres.__dict__ + + def test_roundtrip_http_error(self): + err = tflow.terr() + perr = protobuf._dump_http_error(err) + lerr = protobuf._load_http_error(perr) + assert err.__dict__ == lerr.__dict__ + + def test_roundtrip_http_flow_only_req(self): + f = tflow.tflow() + f.reply = None + pf = protobuf.dumps(f) + lf = protobuf.loads(pf, "http") + assert f.__dict__ == lf.__dict__ + + def test_roundtrip_http_flow_res(self): + f = tflow.tflow(resp=True) + f.reply = None + pf = protobuf.dumps(f) + lf = protobuf.loads(pf, "http") + assert f.__dict__ == lf.__dict__ + + def test_unsupported_dumps(self): + w = tflow.twebsocketflow() + with pytest.raises(exceptions.TypeError): + protobuf.dumps(w) + + def test_unsupported_loads(self): + b = b"blobs" + with pytest.raises(exceptions.TypeError): + protobuf.loads(b, 'not-http') diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py index 2a96995d..b5e226fe 100644 --- a/test/mitmproxy/tools/console/test_commander.py +++ b/test/mitmproxy/tools/console/test_commander.py @@ -28,6 +28,68 @@ class TestListCompleter: assert c.cycle() == expected +class TestCommandHistory: + def fill_history(self, commands): + with taddons.context() as tctx: + history = commander.CommandHistory(tctx.master, size=3) + for c in commands: + cbuf = commander.CommandBuffer(tctx.master, c) + history.add_command(cbuf) + return history, tctx.master + + def test_add_command(self): + commands = ["command1", "command2"] + history, tctx_master = self.fill_history(commands) + + saved_commands = [buf.text for buf in history.saved_commands] + assert saved_commands == [""] + commands + + # The history size is only 3. So, we forget the first + # one command, when adding fourth command + cbuf = commander.CommandBuffer(tctx_master, "command3") + history.add_command(cbuf) + saved_commands = [buf.text for buf in history.saved_commands] + assert saved_commands == commands + ["command3"] + + # Commands with the same text are not repeated in the history one by one + history.add_command(cbuf) + saved_commands = [buf.text for buf in history.saved_commands] + assert saved_commands == commands + ["command3"] + + # adding command in execution mode sets index at the beginning of the history + # and replace the last command buffer if it is empty or has the same text + cbuf = commander.CommandBuffer(tctx_master, "") + history.add_command(cbuf) + history.index = 0 + cbuf = commander.CommandBuffer(tctx_master, "command4") + history.add_command(cbuf, True) + assert history.index == history.last_index + saved_commands = [buf.text for buf in history.saved_commands] + assert saved_commands == ["command2", "command3", "command4"] + + def test_get_next(self): + commands = ["command1", "command2"] + history, tctx_master = self.fill_history(commands) + + history.index = -1 + expected_items = ["", "command1", "command2"] + for i in range(3): + assert history.get_next().text == expected_items[i] + # We are at the last item of the history + assert history.get_next() is None + + def test_get_prev(self): + commands = ["command1", "command2"] + history, tctx_master = self.fill_history(commands) + + expected_items = ["command2", "command1", ""] + history.index = history.last_index + 1 + for i in range(3): + assert history.get_prev().text == expected_items[i] + # We are at the first item of the history + assert history.get_prev() is None + + class TestCommandBuffer: def test_backspace(self): |