aboutsummaryrefslogtreecommitdiffstats
path: root/libmproxy/protocol2
diff options
context:
space:
mode:
authorMaximilian Hils <git@maximilianhils.com>2015-08-30 15:27:29 +0200
committerMaximilian Hils <git@maximilianhils.com>2015-08-30 15:27:29 +0200
commita86ec56012136664688fa4a8efcd866b5e3e17a8 (patch)
treed8aa559db0e3c83a56bc3bac850021f133ad1248 /libmproxy/protocol2
parent421b241ff010ae979cff8df504b6744e4c291aeb (diff)
downloadmitmproxy-a86ec56012136664688fa4a8efcd866b5e3e17a8.tar.gz
mitmproxy-a86ec56012136664688fa4a8efcd866b5e3e17a8.tar.bz2
mitmproxy-a86ec56012136664688fa4a8efcd866b5e3e17a8.zip
move files around
Diffstat (limited to 'libmproxy/protocol2')
-rw-r--r--libmproxy/protocol2/__init__.py13
-rw-r--r--libmproxy/protocol2/http.py588
-rw-r--r--libmproxy/protocol2/http_proxy.py26
-rw-r--r--libmproxy/protocol2/http_replay.py95
-rw-r--r--libmproxy/protocol2/layer.py138
-rw-r--r--libmproxy/protocol2/rawtcp.py66
-rw-r--r--libmproxy/protocol2/reverse_proxy.py17
-rw-r--r--libmproxy/protocol2/root_context.py95
-rw-r--r--libmproxy/protocol2/socks_proxy.py59
-rw-r--r--libmproxy/protocol2/tls.py288
-rw-r--r--libmproxy/protocol2/transparent_proxy.py24
11 files changed, 0 insertions, 1409 deletions
diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py
deleted file mode 100644
index 61b9a77e..00000000
--- a/libmproxy/protocol2/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-from .root_context import RootContext
-from .socks_proxy import Socks5Proxy
-from .reverse_proxy import ReverseProxy
-from .http_proxy import HttpProxy, HttpUpstreamProxy
-from .transparent_proxy import TransparentProxy
-from .http import make_error_response
-
-__all__ = [
- "RootContext",
- "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy",
- "make_error_response"
-]
diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py
deleted file mode 100644
index a508ae8b..00000000
--- a/libmproxy/protocol2/http.py
+++ /dev/null
@@ -1,588 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from netlib import tcp
-from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError
-from netlib.http.semantics import CONTENT_MISSING
-from netlib import odict
-from netlib.tcp import NetLibError, Address
-from netlib.http.http1 import HTTP1Protocol
-from netlib.http.http2 import HTTP2Protocol
-
-from .. import version, utils
-from ..exceptions import InvalidCredentials, HttpException, ProtocolException
-from .layer import Layer
-from ..proxy import Kill
-from libmproxy.protocol import KILL, Error
-from libmproxy.protocol.http import HTTPFlow
-from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest
-
-
-class _HttpLayer(Layer):
- supports_streaming = False
-
- def read_request(self):
- raise NotImplementedError()
-
- def send_request(self, request):
- raise NotImplementedError()
-
- def read_response(self, request_method):
- raise NotImplementedError()
-
- def send_response(self, response):
- raise NotImplementedError()
-
-
-class _StreamingHttpLayer(_HttpLayer):
- supports_streaming = True
-
- def read_response_headers(self):
- raise NotImplementedError
-
- def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
- raise NotImplementedError()
- yield "this is a generator"
-
- def send_response_headers(self, response):
- raise NotImplementedError
-
- def send_response_body(self, response, chunks):
- raise NotImplementedError()
-
-
-class Http1Layer(_StreamingHttpLayer):
- def __init__(self, ctx, mode):
- super(Http1Layer, self).__init__(ctx)
- self.mode = mode
- self.client_protocol = HTTP1Protocol(self.client_conn)
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def read_request(self):
- return HTTPRequest.from_protocol(
- self.client_protocol,
- body_size_limit=self.config.body_size_limit
- )
-
- def send_request(self, request):
- self.server_conn.send(self.server_protocol.assemble(request))
-
- def read_response(self, request_method):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=request_method,
- body_size_limit=self.config.body_size_limit,
- include_body=True
- )
-
- def send_response(self, response):
- self.client_conn.send(self.client_protocol.assemble(response))
-
- def read_response_headers(self):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=None, # does not matter if we don't read the body.
- body_size_limit=self.config.body_size_limit,
- include_body=False
- )
-
- def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
- return self.server_protocol.read_http_body_chunked(
- headers,
- self.config.body_size_limit,
- request_method,
- response_code,
- False,
- max_chunk_size
- )
-
- def send_response_headers(self, response):
- h = self.client_protocol._assemble_response_first_line(response)
- self.client_conn.wfile.write(h + "\r\n")
- h = self.client_protocol._assemble_response_headers(
- response,
- preserve_transfer_encoding=True
- )
- self.client_conn.send(h + "\r\n")
-
- def send_response_body(self, response, chunks):
- if self.client_protocol.has_chunked_encoding(response.headers):
- chunks = (
- "%d\r\n%s\r\n" % (len(chunk), chunk)
- for chunk in chunks
- )
- for chunk in chunks:
- self.client_conn.send(chunk)
-
- def connect(self):
- self.ctx.connect()
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def reconnect(self):
- self.ctx.reconnect()
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def set_server(self, *args, **kwargs):
- self.ctx.set_server(*args, **kwargs)
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def __call__(self):
- layer = HttpLayer(self, self.mode)
- layer()
-
-
-# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
-class Http2Layer(_HttpLayer):
- def __init__(self, ctx, mode):
- super(Http2Layer, self).__init__(ctx)
- self.mode = mode
- self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
-
- def read_request(self):
- request = HTTPRequest.from_protocol(
- self.client_protocol,
- body_size_limit=self.config.body_size_limit
- )
- self._stream_id = request.stream_id
- return request
-
- def send_request(self, message):
- # TODO: implement flow control and WINDOW_UPDATE frames
- self.server_conn.send(self.server_protocol.assemble(message))
-
- def read_response(self, request_method):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=request_method,
- body_size_limit=self.config.body_size_limit,
- include_body=True,
- stream_id=self._stream_id
- )
-
- def send_response(self, message):
- # TODO: implement flow control and WINDOW_UPDATE frames
- self.client_conn.send(self.client_protocol.assemble(message))
-
- def connect(self):
- self.ctx.connect()
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def reconnect(self):
- self.ctx.reconnect()
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def set_server(self, *args, **kwargs):
- self.ctx.set_server(*args, **kwargs)
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def __call__(self):
- self.server_protocol.perform_connection_preface()
- layer = HttpLayer(self, self.mode)
- layer()
-
- def handle_unexpected_frame(self, frm):
- self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info")
-
-
-def make_error_response(status_code, message, headers=None):
- response = status_codes.RESPONSES.get(status_code, "Unknown")
- body = """
- <html>
- <head>
- <title>%d %s</title>
- </head>
- <body>%s</body>
- </html>
- """.strip() % (status_code, response, message)
-
- if not headers:
- headers = odict.ODictCaseless()
- headers["Server"] = [version.NAMEVERSION]
- headers["Connection"] = ["close"]
- headers["Content-Length"] = [len(body)]
- headers["Content-Type"] = ["text/html"]
-
- return HTTPResponse(
- (1, 1), # FIXME: Should be a string.
- status_code,
- response,
- headers,
- body,
- )
-
-
-def make_connect_request(address):
- address = Address.wrap(address)
- return HTTPRequest(
- "authority", "CONNECT", None, address.host, address.port, None, (1, 1),
- odict.ODictCaseless(), ""
- )
-
-
-def make_connect_response(httpversion):
- headers = odict.ODictCaseless([
- ["Content-Length", "0"],
- ["Proxy-Agent", version.NAMEVERSION]
- ])
- return HTTPResponse(
- httpversion,
- 200,
- "Connection established",
- headers,
- "",
- )
-
-
-class ConnectServerConnection(object):
- """
- "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
- """
-
- def __init__(self, address, ctx):
- self.address = tcp.Address.wrap(address)
- self._ctx = ctx
-
- @property
- def via(self):
- return self._ctx.server_conn
-
- def __getattr__(self, item):
- return getattr(self.via, item)
-
-
-class UpstreamConnectLayer(Layer):
- def __init__(self, ctx, connect_request):
- super(UpstreamConnectLayer, self).__init__(ctx)
- self.connect_request = connect_request
- self.server_conn = ConnectServerConnection(
- (connect_request.host, connect_request.port),
- self.ctx
- )
-
- def __call__(self):
- layer = self.ctx.next_layer(self)
- layer()
-
- def connect(self):
- if not self.server_conn:
- self.ctx.connect()
- self.send_request(self.connect_request)
- else:
- pass # swallow the message
-
- def reconnect(self):
- self.ctx.reconnect()
- self.send_request(self.connect_request)
- resp = self.read_response("CONNECT")
- if resp.code != 200:
- raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
-
- def set_server(self, address, server_tls=None, sni=None, depth=1):
- if depth == 1:
- if self.ctx.server_conn:
- self.ctx.reconnect()
- address = Address.wrap(address)
- self.connect_request.host = address.host
- self.connect_request.port = address.port
- self.server_conn.address = address
- else:
- self.ctx.set_server(address, server_tls, sni, depth - 1)
-
-
-class HttpLayer(Layer):
- def __init__(self, ctx, mode):
- super(HttpLayer, self).__init__(ctx)
- self.mode = mode
- self.__original_server_conn = None
- "Contains the original destination in transparent mode, which needs to be restored"
- "if an inline script modified the target server for a single http request"
-
- def __call__(self):
- if self.mode == "transparent":
- self.__original_server_conn = self.server_conn
- while True:
- try:
- flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
-
- try:
- request = self.read_request()
- except tcp.NetLibError:
- # don't throw an error for disconnects that happen
- # before/between requests.
- return
-
- self.log("request", "debug", [repr(request)])
-
- # Handle Proxy Authentication
- self.authenticate(request)
-
- # Regular Proxy Mode: Handle CONNECT
- if self.mode == "regular" and request.form_in == "authority":
- self.handle_regular_mode_connect(request)
- return
-
- # Make sure that the incoming request matches our expectations
- self.validate_request(request)
-
- flow.request = request
- self.process_request_hook(flow)
-
- if not flow.response:
- self.establish_server_connection(flow)
- self.get_response_from_server(flow)
-
- self.send_response_to_client(flow)
-
- if self.check_close_connection(flow):
- return
-
- # TODO: Implement HTTP Upgrade
-
- # Upstream Proxy Mode: Handle CONNECT
- if flow.request.form_in == "authority" and flow.response.code == 200:
- self.handle_upstream_mode_connect(flow.request.copy())
- return
-
- except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
- if flow.request and not flow.response:
- flow.error = Error(repr(e))
- self.channel.ask("error", flow)
- try:
- self.send_response(make_error_response(
- getattr(e, "code", 502),
- repr(e)
- ))
- except NetLibError:
- pass
- if isinstance(e, ProtocolException):
- raise e
- else:
- raise ProtocolException("Error in HTTP connection: %s" % repr(e), e)
- finally:
- flow.live = False
-
- def handle_regular_mode_connect(self, request):
- self.set_server((request.host, request.port))
- self.send_response(make_connect_response(request.httpversion))
- layer = self.ctx.next_layer(self)
- layer()
-
- def handle_upstream_mode_connect(self, connect_request):
- layer = UpstreamConnectLayer(self, connect_request)
- layer()
-
- def check_close_connection(self, flow):
- """
- Checks if the connection should be closed depending on the HTTP
- semantics. Returns True, if so.
- """
-
- # TODO: add logic for HTTP/2
-
- close_connection = (
- http1.HTTP1Protocol.connection_close(
- flow.request.httpversion,
- flow.request.headers
- ) or http1.HTTP1Protocol.connection_close(
- flow.response.httpversion,
- flow.response.headers
- ) or http1.HTTP1Protocol.expected_http_body_size(
- flow.response.headers,
- False,
- flow.request.method,
- flow.response.code) == -1
- )
- if flow.request.form_in == "authority" and flow.response.code == 200:
- # Workaround for
- # https://github.com/mitmproxy/mitmproxy/issues/313: Some
- # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
- # and no Content-Length header
-
- return False
- return close_connection
-
- def send_response_to_client(self, flow):
- if not (self.supports_streaming and flow.response.stream):
- # no streaming:
- # we already received the full response from the server and can
- # send it to the client straight away.
- self.send_response(flow.response)
- else:
- # streaming:
- # First send the headers and then transfer the response incrementally
- self.send_response_headers(flow.response)
- chunks = self.read_response_body(
- flow.response.headers,
- flow.request.method,
- flow.response.code,
- max_chunk_size=4096
- )
- if callable(flow.response.stream):
- chunks = flow.response.stream(chunks)
- self.send_response_body(flow.response, chunks)
- flow.response.timestamp_end = utils.timestamp()
-
- def get_response_from_server(self, flow):
- def get_response():
- self.send_request(flow.request)
- if self.supports_streaming:
- flow.response = self.read_response_headers()
- else:
- flow.response = self.read_response()
-
- try:
- get_response()
- except (tcp.NetLibError, HttpErrorConnClosed) as v:
- self.log(
- "server communication error: %s" % repr(v),
- level="debug"
- )
- # In any case, we try to reconnect at least once. This is
- # necessary because it might be possible that we already
- # initiated an upstream connection after clientconnect that
- # has already been expired, e.g consider the following event
- # log:
- # > clientconnect (transparent mode destination known)
- # > serverconnect (required for client tls handshake)
- # > read n% of large request
- # > server detects timeout, disconnects
- # > read (100-n)% of large request
- # > send large request upstream
- self.reconnect()
- get_response()
-
- # call the appropriate script hook - this is an opportunity for an
- # inline script to set flow.stream = True
- flow = self.channel.ask("responseheaders", flow)
- if flow is None or flow == KILL:
- raise Kill()
-
- if self.supports_streaming:
- if flow.response.stream:
- flow.response.content = CONTENT_MISSING
- else:
- flow.response.content = "".join(self.read_response_body(
- flow.response.headers,
- flow.request.method,
- flow.response.code
- ))
- flow.response.timestamp_end = utils.timestamp()
-
- # no further manipulation of self.server_conn beyond this point
- # we can safely set it as the final attribute value here.
- flow.server_conn = self.server_conn
-
- self.log(
- "response",
- "debug",
- [repr(flow.response)]
- )
- response_reply = self.channel.ask("response", flow)
- if response_reply is None or response_reply == KILL:
- raise Kill()
-
- def process_request_hook(self, flow):
- # Determine .scheme, .host and .port attributes for inline scripts.
- # For absolute-form requests, they are directly given in the request.
- # For authority-form requests, we only need to determine the request scheme.
- # For relative-form requests, we need to determine host and port as
- # well.
- if self.mode == "regular":
- pass # only absolute-form at this point, nothing to do here.
- elif self.mode == "upstream":
- if flow.request.form_in == "authority":
- flow.request.scheme = "http" # pseudo value
- else:
- flow.request.host = self.__original_server_conn.address.host
- flow.request.port = self.__original_server_conn.address.port
- flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http"
-
- request_reply = self.channel.ask("request", flow)
- if request_reply is None or request_reply == KILL:
- raise Kill()
- if isinstance(request_reply, HTTPResponse):
- flow.response = request_reply
- return
-
- def establish_server_connection(self, flow):
- address = tcp.Address((flow.request.host, flow.request.port))
- tls = (flow.request.scheme == "https")
-
- if self.mode == "regular" or self.mode == "transparent":
- # If there's an existing connection that doesn't match our expectations, kill it.
- if address != self.server_conn.address or tls != self.server_conn.ssl_established:
- self.set_server(address, tls, address.host)
- # Establish connection is neccessary.
- if not self.server_conn:
- self.connect()
-
- # SetServer is not guaranteed to work with TLS:
- # If there's not TlsLayer below which could catch the exception,
- # TLS will not be established.
- if tls and not self.server_conn.tls_established:
- raise ProtocolException(
- "Cannot upgrade to SSL, no TLS layer on the protocol stack.")
- else:
- if not self.server_conn:
- self.connect()
- if tls:
- raise HttpException("Cannot change scheme in upstream proxy mode.")
- """
- # This is a very ugly (untested) workaround to solve a very ugly problem.
- if self.server_conn and self.server_conn.tls_established and not ssl:
- self.reconnect()
- elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
- if self.server_conn.tls_established:
- self.reconnect()
-
- self.send_request(make_connect_request(address))
- tls_layer = TlsLayer(self, False, True)
- tls_layer._establish_tls_with_server()
- """
-
- def validate_request(self, request):
- if request.form_in == "absolute" and request.scheme != "http":
- self.send_response(
- make_error_response(400, "Invalid request scheme: %s" % request.scheme))
- raise HttpException("Invalid request scheme: %s" % request.scheme)
-
- expected_request_forms = {
- "regular": ("absolute",), # an authority request would already be handled.
- "upstream": ("authority", "absolute"),
- "transparent": ("relative",)
- }
-
- allowed_request_forms = expected_request_forms[self.mode]
- if request.form_in not in allowed_request_forms:
- err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
- " or ".join(allowed_request_forms), request.form_in
- )
- self.send_response(make_error_response(400, err_message))
- raise HttpException(err_message)
-
- if self.mode == "regular":
- request.form_out = "relative"
-
- def authenticate(self, request):
- if self.config.authenticator:
- if self.config.authenticator.authenticate(request.headers):
- self.config.authenticator.clean(request.headers)
- else:
- self.send_response(make_error_response(
- 407,
- "Proxy Authentication Required",
- odict.ODictCaseless(
- [
- [k, v] for k, v in
- self.config.authenticator.auth_challenge_headers().items()
- ])
- ))
- raise InvalidCredentials("Proxy Authentication Required")
diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py
deleted file mode 100644
index 2876c022..00000000
--- a/libmproxy/protocol2/http_proxy.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from .layer import Layer, ServerConnectionMixin
-
-
-class HttpProxy(Layer, ServerConnectionMixin):
- def __call__(self):
- layer = self.ctx.next_layer(self)
- try:
- layer()
- finally:
- if self.server_conn:
- self._disconnect()
-
-
-class HttpUpstreamProxy(Layer, ServerConnectionMixin):
- def __init__(self, ctx, server_address):
- super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address)
-
- def __call__(self):
- layer = self.ctx.next_layer(self)
- try:
- layer()
- finally:
- if self.server_conn:
- self._disconnect()
diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py
deleted file mode 100644
index 872ef9cd..00000000
--- a/libmproxy/protocol2/http_replay.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import threading
-from netlib.http import HttpError
-from netlib.http.http1 import HTTP1Protocol
-from netlib.tcp import NetLibError
-
-from ..controller import Channel
-from ..protocol import KILL, Error
-from ..protocol.http_wrappers import HTTPResponse
-from ..proxy import Log, Kill
-from ..proxy.connection import ServerConnection
-from .http import make_connect_request
-
-
-class RequestReplayThread(threading.Thread):
- name = "RequestReplayThread"
-
- def __init__(self, config, flow, masterq, should_exit):
- """
- masterqueue can be a queue or None, if no scripthooks should be
- processed.
- """
- self.config, self.flow = config, flow
- if masterq:
- self.channel = Channel(masterq, should_exit)
- else:
- self.channel = None
- super(RequestReplayThread, self).__init__()
-
- def run(self):
- r = self.flow.request
- form_out_backup = r.form_out
- try:
- self.flow.response = None
-
- # If we have a channel, run script hooks.
- if self.channel:
- request_reply = self.channel.ask("request", self.flow)
- if request_reply is None or request_reply == KILL:
- raise Kill()
- elif isinstance(request_reply, HTTPResponse):
- self.flow.response = request_reply
-
- if not self.flow.response:
- # In all modes, we directly connect to the server displayed
- if self.config.mode == "upstream":
- server_address = self.config.upstream_server.address
- server = ServerConnection(server_address)
- server.connect()
- protocol = HTTP1Protocol(server)
- if r.scheme == "https":
- connect_request = make_connect_request((r.host, r.port))
- server.send(protocol.assemble(connect_request))
- resp = protocol.read_response("CONNECT")
- if resp.code != 200:
- raise HttpError(502, "Upstream server refuses CONNECT request")
- server.establish_ssl(
- self.config.clientcerts,
- sni=self.flow.server_conn.sni
- )
- r.form_out = "relative"
- else:
- r.form_out = "absolute"
- else:
- server_address = (r.host, r.port)
- server = ServerConnection(server_address)
- server.connect()
- protocol = HTTP1Protocol(server)
- if r.scheme == "https":
- server.establish_ssl(
- self.config.clientcerts,
- sni=self.flow.server_conn.sni
- )
- r.form_out = "relative"
-
- server.send(protocol.assemble(r))
- self.flow.server_conn = server
- self.flow.response = HTTPResponse.from_protocol(
- protocol,
- r.method,
- body_size_limit=self.config.body_size_limit,
- )
- if self.channel:
- response_reply = self.channel.ask("response", self.flow)
- if response_reply is None or response_reply == KILL:
- raise Kill()
- except (HttpError, NetLibError) as v:
- self.flow.error = Error(repr(v))
- if self.channel:
- self.channel.ask("error", self.flow)
- except Kill:
- # KillSignal should only be raised if there's a channel in the
- # first place.
- self.channel.tell("log", Log("Connection killed", "info"))
- finally:
- r.form_out = form_out_backup
diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py
deleted file mode 100644
index 2b47cc26..00000000
--- a/libmproxy/protocol2/layer.py
+++ /dev/null
@@ -1,138 +0,0 @@
-"""
-mitmproxy protocol architecture
-
-In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other.
-For example, the following scenarios depict possible scenarios (lowest layer first):
-
-Transparent HTTP proxy, no SSL:
- TransparentModeLayer
- HttpLayer
-
-Regular proxy, CONNECT request with WebSockets over SSL:
- RegularModeLayer
- HttpLayer
- SslLayer
- WebsocketLayer (or TcpLayer)
-
-Automated protocol detection by peeking into the buffer:
- TransparentModeLayer
- SslLayer
- Http2Layer
-
-Communication between layers is done as follows:
- - lower layers provide context information to higher layers
- - higher layers can call functions provided by lower layers,
- which are propagated until they reach a suitable layer.
-
-Further goals:
- - Connections should always be peekable to make automatic protocol detection work.
- - Upstream connections should be established as late as possible;
- inline scripts shall have a chance to handle everything locally.
-"""
-from __future__ import (absolute_import, print_function, division)
-from netlib import tcp
-from ..proxy import Log
-from ..proxy.connection import ServerConnection
-from ..exceptions import ProtocolException
-
-
-class _LayerCodeCompletion(object):
- """
- Dummy class that provides type hinting in PyCharm, which simplifies development a lot.
- """
-
- def __init__(self, *args, **kwargs):
- super(_LayerCodeCompletion, self).__init__(*args, **kwargs)
- if True:
- return
- self.config = None
- """@type: libmproxy.proxy.config.ProxyConfig"""
- self.client_conn = None
- """@type: libmproxy.proxy.connection.ClientConnection"""
- self.channel = None
- """@type: libmproxy.controller.Channel"""
-
-
-class Layer(_LayerCodeCompletion):
- def __init__(self, ctx, *args, **kwargs):
- """
- Args:
- ctx: The (read-only) higher layer.
- """
- super(Layer, self).__init__(*args, **kwargs)
- self.ctx = ctx
-
- def __call__(self):
- """
- Logic of the layer.
- Raises:
- ProtocolException in case of protocol exceptions.
- """
- raise NotImplementedError
-
- def __getattr__(self, name):
- """
- Attributes not present on the current layer may exist on a higher layer.
- """
- return getattr(self.ctx, name)
-
- def log(self, msg, level, subs=()):
- full_msg = [
- "{}: {}".format(repr(self.client_conn.address), msg)
- ]
- for i in subs:
- full_msg.append(" -> " + i)
- full_msg = "\n".join(full_msg)
- self.channel.tell("log", Log(full_msg, level))
-
- @property
- def layers(self):
- return [self] + self.ctx.layers
-
- def __repr__(self):
- return type(self).__name__
-
-
-class ServerConnectionMixin(object):
- """
- Mixin that provides a layer with the capabilities to manage a server connection.
- """
-
- def __init__(self, server_address=None):
- super(ServerConnectionMixin, self).__init__()
- self.server_conn = ServerConnection(server_address)
-
- def reconnect(self):
- address = self.server_conn.address
- self._disconnect()
- self.server_conn.address = address
- self.connect()
-
- def set_server(self, address, server_tls=None, sni=None, depth=1):
- if depth == 1:
- if self.server_conn:
- self._disconnect()
- self.log("Set new server address: " + repr(address), "debug")
- self.server_conn.address = address
- else:
- self.ctx.set_server(address, server_tls, sni, depth - 1)
-
- def _disconnect(self):
- """
- Deletes (and closes) an existing server connection.
- """
- self.log("serverdisconnect", "debug", [repr(self.server_conn.address)])
- self.server_conn.finish()
- self.server_conn.close()
- # self.channel.tell("serverdisconnect", self)
- self.server_conn = ServerConnection(None)
-
- def connect(self):
- if not self.server_conn.address:
- raise ProtocolException("Cannot connect to server, no server address given.")
- self.log("serverconnect", "debug", [repr(self.server_conn.address)])
- try:
- self.server_conn.connect()
- except tcp.NetLibError as e:
- raise ProtocolException(
- "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e)
diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py
deleted file mode 100644
index b10217f1..00000000
--- a/libmproxy/protocol2/rawtcp.py
+++ /dev/null
@@ -1,66 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-import socket
-import select
-
-from OpenSSL import SSL
-
-from netlib.tcp import NetLibError
-from netlib.utils import cleanBin
-from ..exceptions import ProtocolException
-from .layer import Layer
-
-
-class RawTcpLayer(Layer):
- chunk_size = 4096
-
- def __init__(self, ctx, logging=True):
- self.logging = logging
- super(RawTcpLayer, self).__init__(ctx)
-
- def __call__(self):
- self.connect()
-
- buf = memoryview(bytearray(self.chunk_size))
-
- client = self.client_conn.connection
- server = self.server_conn.connection
- conns = [client, server]
-
- try:
- while True:
- r, _, _ = select.select(conns, [], [], 10)
- for conn in r:
- dst = server if conn == client else client
-
- size = conn.recv_into(buf, self.chunk_size)
- if not size:
- conns.remove(conn)
- # Shutdown connection to the other peer
- if isinstance(conn, SSL.Connection):
- # We can't half-close a connection, so we just close everything here.
- # Sockets will be cleaned up on a higher level.
- return
- else:
- dst.shutdown(socket.SHUT_WR)
-
- if len(conns) == 0:
- return
- continue
-
- dst.sendall(buf[:size])
-
- if self.logging:
- # log messages are prepended with the client address,
- # hence the "weird" direction string.
- if dst == server:
- direction = "-> tcp -> {}".format(repr(self.server_conn.address))
- else:
- direction = "<- tcp <- {}".format(repr(self.server_conn.address))
- data = cleanBin(buf[:size].tobytes())
- self.log(
- "{}\r\n{}".format(direction, data),
- "info"
- )
-
- except (socket.error, NetLibError, SSL.Error) as e:
- raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e)
diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py
deleted file mode 100644
index 3ca998d5..00000000
--- a/libmproxy/protocol2/reverse_proxy.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from .layer import Layer, ServerConnectionMixin
-
-
-class ReverseProxy(Layer, ServerConnectionMixin):
- def __init__(self, ctx, server_address, server_tls):
- super(ReverseProxy, self).__init__(ctx, server_address=server_address)
- self.server_tls = server_tls
-
- def __call__(self):
- layer = self.ctx.next_layer(self)
- try:
- layer()
- finally:
- if self.server_conn:
- self._disconnect()
diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py
deleted file mode 100644
index daea54bd..00000000
--- a/libmproxy/protocol2/root_context.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from netlib.http.http1 import HTTP1Protocol
-from netlib.http.http2 import HTTP2Protocol
-
-from .rawtcp import RawTcpLayer
-from .tls import TlsLayer, is_tls_record_magic
-from .http import Http1Layer, Http2Layer
-from .layer import ServerConnectionMixin
-from .http_proxy import HttpProxy, HttpUpstreamProxy
-from .reverse_proxy import ReverseProxy
-
-
-class RootContext(object):
- """
- The outmost context provided to the root layer.
- As a consequence, every layer has .client_conn, .channel, .next_layer() and .config.
- """
-
- def __init__(self, client_conn, config, channel):
- self.client_conn = client_conn # Client Connection
- self.channel = channel # provides .ask() method to communicate with FlowMaster
- self.config = config # Proxy Configuration
-
- def next_layer(self, top_layer):
- """
- This function determines the next layer in the protocol stack.
-
- Arguments:
- top_layer: the current top layer.
-
- Returns:
- The next layer
- """
-
- # 1. Check for --ignore.
- if self.config.check_ignore(top_layer.server_conn.address):
- return RawTcpLayer(top_layer, logging=False)
-
- d = top_layer.client_conn.rfile.peek(3)
- client_tls = is_tls_record_magic(d)
-
- # 2. Always insert a TLS layer, even if there's neither client nor server tls.
- # An inline script may upgrade from http to https,
- # in which case we need some form of TLS layer.
- if isinstance(top_layer, ReverseProxy):
- return TlsLayer(top_layer, client_tls, top_layer.server_tls)
- if isinstance(top_layer, ServerConnectionMixin):
- return TlsLayer(top_layer, client_tls, client_tls)
-
- # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
- if isinstance(top_layer, TlsLayer):
- if isinstance(top_layer.ctx, HttpProxy):
- return Http1Layer(top_layer, "regular")
- if isinstance(top_layer.ctx, HttpUpstreamProxy):
- return Http1Layer(top_layer, "upstream")
-
- # 4. Check for other TLS cases (e.g. after CONNECT).
- if client_tls:
- return TlsLayer(top_layer, True, True)
-
- # 4. Check for --tcp
- if self.config.check_tcp(top_layer.server_conn.address):
- return RawTcpLayer(top_layer)
-
- # 5. Check for TLS ALPN (HTTP1/HTTP2)
- if isinstance(top_layer, TlsLayer):
- alpn = top_layer.client_conn.get_alpn_proto_negotiated()
- if alpn == HTTP2Protocol.ALPN_PROTO_H2:
- return Http2Layer(top_layer, 'transparent')
- if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1:
- return Http1Layer(top_layer, 'transparent')
-
- # 6. Assume HTTP1 by default
- return Http1Layer(top_layer, 'transparent')
-
- # In a future version, we want to implement TCP passthrough as the last fallback,
- # but we don't have the UI part ready for that.
- #
- # d = top_layer.client_conn.rfile.peek(3)
- # is_ascii = (
- # len(d) == 3 and
- # # better be safe here and don't expect uppercase...
- # all(x in string.ascii_letters for x in d)
- # )
- # # TODO: This could block if there are not enough bytes available?
- # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE))
- # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE)
-
- @property
- def layers(self):
- return []
-
- def __repr__(self):
- return "RootContext"
diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py
deleted file mode 100644
index 525520e8..00000000
--- a/libmproxy/protocol2/socks_proxy.py
+++ /dev/null
@@ -1,59 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from netlib import socks
-from netlib.tcp import NetLibError
-from ..exceptions import Socks5Exception
-from .layer import Layer, ServerConnectionMixin
-
-
-class Socks5Proxy(Layer, ServerConnectionMixin):
- def __call__(self):
- try:
- # Parse Client Greeting
- client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True)
- client_greet.assert_socks5()
- if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods:
- raise socks.SocksError(
- socks.METHOD.NO_ACCEPTABLE_METHODS,
- "mitmproxy only supports SOCKS without authentication"
- )
-
- # Send Server Greeting
- server_greet = socks.ServerGreeting(
- socks.VERSION.SOCKS5,
- socks.METHOD.NO_AUTHENTICATION_REQUIRED
- )
- server_greet.to_file(self.client_conn.wfile)
- self.client_conn.wfile.flush()
-
- # Parse Connect Request
- connect_request = socks.Message.from_file(self.client_conn.rfile)
- connect_request.assert_socks5()
- if connect_request.msg != socks.CMD.CONNECT:
- raise socks.SocksError(
- socks.REP.COMMAND_NOT_SUPPORTED,
- "mitmproxy only supports SOCKS5 CONNECT."
- )
-
- # We always connect lazily, but we need to pretend to the client that we connected.
- connect_reply = socks.Message(
- socks.VERSION.SOCKS5,
- socks.REP.SUCCEEDED,
- connect_request.atyp,
- # dummy value, we don't have an upstream connection yet.
- connect_request.addr
- )
- connect_reply.to_file(self.client_conn.wfile)
- self.client_conn.wfile.flush()
-
- except (socks.SocksError, NetLibError) as e:
- raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e)
-
- self.server_conn.address = connect_request.addr
-
- layer = self.ctx.next_layer(self)
- try:
- layer()
- finally:
- if self.server_conn:
- self._disconnect()
diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py
deleted file mode 100644
index 73bb12f3..00000000
--- a/libmproxy/protocol2/tls.py
+++ /dev/null
@@ -1,288 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-import struct
-
-from construct import ConstructError
-
-from netlib.tcp import NetLibError, NetLibInvalidCertificateError
-from netlib.http.http1 import HTTP1Protocol
-from ..contrib.tls._constructs import ClientHello
-from ..exceptions import ProtocolException
-from .layer import Layer
-
-
-def is_tls_record_magic(d):
- """
- Returns:
- True, if the passed bytes start with the TLS record magic bytes.
- False, otherwise.
- """
- d = d[:3]
-
- # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2
- # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
- return (
- len(d) == 3 and
- d[0] == '\x16' and
- d[1] == '\x03' and
- d[2] in ('\x00', '\x01', '\x02', '\x03')
- )
-
-
-class TlsLayer(Layer):
- def __init__(self, ctx, client_tls, server_tls):
- self.client_sni = None
- self.client_alpn_protocols = None
-
- super(TlsLayer, self).__init__(ctx)
- self._client_tls = client_tls
- self._server_tls = server_tls
-
- self._sni_from_server_change = None
-
- def __call__(self):
- """
- The strategy for establishing SSL is as follows:
- First, we determine whether we need the server cert to establish ssl with the client.
- If so, we first connect to the server and then to the client.
- If not, we only connect to the client and do the server_ssl lazily on a Connect message.
-
- An additional complexity is that establish ssl with the server may require a SNI value from the client.
- In an ideal world, we'd do the following:
- 1. Start the SSL handshake with the client
- 2. Check if the client sends a SNI.
- 3. Pause the client handshake, establish SSL with the server.
- 4. Finish the client handshake with the certificate from the server.
- There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :(
- Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values.
-
- Further notes:
- - OpenSSL 1.0.2 introduces a callback that would help here:
- https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html
- - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427
- """
-
- client_tls_requires_server_cert = (
- self._client_tls and self._server_tls and not self.config.no_upstream_cert
- )
-
- if self._client_tls:
- self._parse_client_hello()
-
- if client_tls_requires_server_cert:
- self._establish_tls_with_client_and_server()
- elif self._client_tls:
- self._establish_tls_with_client()
-
- layer = self.ctx.next_layer(self)
- layer()
-
- def __repr__(self):
- if self._client_tls and self._server_tls:
- return "TlsLayer(client and server)"
- elif self._client_tls:
- return "TlsLayer(client)"
- elif self._server_tls:
- return "TlsLayer(server)"
- else:
- return "TlsLayer(inactive)"
-
- def _get_client_hello(self):
- """
- Peek into the socket and read all records that contain the initial client hello message.
-
- Returns:
- The raw handshake packet bytes, without TLS record header(s).
- """
- client_hello = ""
- client_hello_size = 1
- offset = 0
- while len(client_hello) < client_hello_size:
- record_header = self.client_conn.rfile.peek(offset + 5)[offset:]
- if not is_tls_record_magic(record_header) or len(record_header) != 5:
- raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header)
- record_size = struct.unpack("!H", record_header[3:])[0] + 5
- record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:]
- if len(record_body) != record_size - 5:
- raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
- client_hello += record_body
- offset += record_size
- client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4
- return client_hello
-
- def _parse_client_hello(self):
- """
- Peek into the connection, read the initial client hello and parse it to obtain ALPN values.
- """
- try:
- raw_client_hello = self._get_client_hello()[4:] # exclude handshake header.
- except ProtocolException as e:
- self.log("Cannot parse Client Hello: %s" % repr(e), "error")
- return
-
- try:
- client_hello = ClientHello.parse(raw_client_hello)
- except ConstructError as e:
- self.log("Cannot parse Client Hello: %s" % repr(e), "error")
- self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug")
- return
-
- for extension in client_hello.extensions:
- if extension.type == 0x00:
- if len(extension.server_names) != 1 or extension.server_names[0].type != 0:
- self.log("Unknown Server Name Indication: %s" % extension.server_names, "error")
- self.client_sni = extension.server_names[0].name
- elif extension.type == 0x10:
- self.client_alpn_protocols = list(extension.alpn_protocols)
-
- self.log(
- "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols),
- "debug"
- )
-
- def connect(self):
- if not self.server_conn:
- self.ctx.connect()
- if self._server_tls and not self.server_conn.tls_established:
- self._establish_tls_with_server()
-
- def reconnect(self):
- self.ctx.reconnect()
- if self._server_tls and not self.server_conn.tls_established:
- self._establish_tls_with_server()
-
- def set_server(self, address, server_tls=None, sni=None, depth=1):
- self.ctx.set_server(address, server_tls, sni, depth)
- if depth == 1 and server_tls is not None:
- self._sni_from_server_change = sni
- self._server_tls = server_tls
-
- @property
- def sni_for_server_connection(self):
- if self._sni_from_server_change is False:
- return None
- else:
- return self._sni_from_server_change or self.client_sni
-
- @property
- def alpn_for_client_connection(self):
- return self.server_conn.get_alpn_proto_negotiated()
-
- def __alpn_select_callback(self, conn_, options):
- """
- Once the client signals the alternate protocols it supports,
- we reconnect upstream with the same list and pass the server's choice down to the client.
- """
-
- # This gets triggered if we haven't established an upstream connection yet.
- default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1
- # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2
-
- if self.alpn_for_client_connection in options:
- choice = bytes(self.alpn_for_client_connection)
- elif default_alpn in options:
- choice = bytes(default_alpn)
- else:
- choice = options[0]
- self.log("ALPN for client: %s" % choice, "debug")
- return choice
-
- def _establish_tls_with_client_and_server(self):
- self.ctx.connect()
-
- # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless
- # to send an error message over TLS.
- try:
- self._establish_tls_with_server()
- except Exception as e:
- try:
- self._establish_tls_with_client()
- except:
- pass
- raise e
-
- self._establish_tls_with_client()
-
- def _establish_tls_with_client(self):
- self.log("Establish TLS with client", "debug")
- cert, key, chain_file = self._find_cert()
-
- try:
- self.client_conn.convert_to_ssl(
- cert, key,
- method=self.config.openssl_method_client,
- options=self.config.openssl_options_client,
- cipher_list=self.config.ciphers_client,
- dhparams=self.config.certstore.dhparams,
- chain_file=chain_file,
- alpn_select_callback=self.__alpn_select_callback,
- )
- except NetLibError as e:
- raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e)
-
- def _establish_tls_with_server(self):
- self.log("Establish TLS with server", "debug")
- try:
- # We only support http/1.1 and h2.
- # If the server only supports spdy (next to http/1.1), it may select that
- # and mitmproxy would enter TCP passthrough mode, which we want to avoid.
- deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy")
- if self.client_alpn_protocols:
- alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols)
- else:
- alpn = None
-
- self.server_conn.establish_ssl(
- self.config.clientcerts,
- self.sni_for_server_connection,
- method=self.config.openssl_method_server,
- options=self.config.openssl_options_server,
- verify_options=self.config.openssl_verification_mode_server,
- ca_path=self.config.openssl_trusted_cadir_server,
- ca_pemfile=self.config.openssl_trusted_ca_server,
- cipher_list=self.config.ciphers_server,
- alpn_protos=alpn,
- )
- tls_cert_err = self.server_conn.ssl_verification_error
- if tls_cert_err is not None:
- self.log(
- "TLS verification failed for upstream server at depth %s with error: %s" %
- (tls_cert_err['depth'], tls_cert_err['errno']),
- "error")
- self.log("Ignoring server verification error, continuing with connection", "error")
- except NetLibInvalidCertificateError as e:
- tls_cert_err = self.server_conn.ssl_verification_error
- self.log(
- "TLS verification failed for upstream server at depth %s with error: %s" %
- (tls_cert_err['depth'], tls_cert_err['errno']),
- "error")
- self.log("Aborting connection attempt", "error")
- raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e)
- except NetLibError as e:
- raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e)
-
- self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug")
-
- def _find_cert(self):
- host = self.server_conn.address.host
- sans = set()
- # Incorporate upstream certificate
- use_upstream_cert = (
- self.server_conn and
- self.server_conn.tls_established and
- (not self.config.no_upstream_cert)
- )
- if use_upstream_cert:
- upstream_cert = self.server_conn.cert
- sans.update(upstream_cert.altnames)
- if upstream_cert.cn:
- sans.add(host)
- host = upstream_cert.cn.decode("utf8").encode("idna")
- # Also add SNI values.
- if self.client_sni:
- sans.add(self.client_sni)
- if self._sni_from_server_change:
- sans.add(self._sni_from_server_change)
-
- sans.discard(host)
- return self.config.certstore.get_cert(host, list(sans))
diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py
deleted file mode 100644
index e6ebf115..00000000
--- a/libmproxy/protocol2/transparent_proxy.py
+++ /dev/null
@@ -1,24 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from ..exceptions import ProtocolException
-from .. import platform
-from .layer import Layer, ServerConnectionMixin
-
-
-class TransparentProxy(Layer, ServerConnectionMixin):
- def __init__(self, ctx):
- super(TransparentProxy, self).__init__(ctx)
- self.resolver = platform.resolver()
-
- def __call__(self):
- try:
- self.server_conn.address = self.resolver.original_addr(self.client_conn.connection)
- except Exception as e:
- raise ProtocolException("Transparent mode failure: %s" % repr(e), e)
-
- layer = self.ctx.next_layer(self)
- try:
- layer()
- finally:
- if self.server_conn:
- self._disconnect()