aboutsummaryrefslogtreecommitdiffstats
path: root/libmproxy
diff options
context:
space:
mode:
Diffstat (limited to 'libmproxy')
-rw-r--r--libmproxy/console/common.py2
-rw-r--r--libmproxy/console/flowview.py2
-rw-r--r--libmproxy/filt.py2
-rw-r--r--libmproxy/flow.py23
-rw-r--r--libmproxy/models/__init__.py16
-rw-r--r--libmproxy/models/connections.py (renamed from libmproxy/proxy/connection.py)1
-rw-r--r--libmproxy/models/flow.py (renamed from libmproxy/protocol/primitives.py)8
-rw-r--r--libmproxy/models/http.py (renamed from libmproxy/protocol/http_wrappers.py)211
-rw-r--r--libmproxy/protocol/__init__.py13
-rw-r--r--libmproxy/protocol/base.py (renamed from libmproxy/protocol2/layer.py)28
-rw-r--r--libmproxy/protocol/http.py602
-rw-r--r--libmproxy/protocol/http_replay.py (renamed from libmproxy/protocol2/http_replay.py)16
-rw-r--r--libmproxy/protocol/rawtcp.py (renamed from libmproxy/protocol2/rawtcp.py)6
-rw-r--r--libmproxy/protocol/tls.py (renamed from libmproxy/protocol2/tls.py)2
-rw-r--r--libmproxy/protocol2/http.py588
-rw-r--r--libmproxy/proxy/__init__.py8
-rw-r--r--libmproxy/proxy/config.py2
-rw-r--r--libmproxy/proxy/modes/__init__.py (renamed from libmproxy/protocol2/__init__.py)13
-rw-r--r--libmproxy/proxy/modes/http_proxy.py (renamed from libmproxy/protocol2/http_proxy.py)2
-rw-r--r--libmproxy/proxy/modes/reverse_proxy.py (renamed from libmproxy/protocol2/reverse_proxy.py)2
-rw-r--r--libmproxy/proxy/modes/socks_proxy.py (renamed from libmproxy/protocol2/socks_proxy.py)5
-rw-r--r--libmproxy/proxy/modes/transparent_proxy.py (renamed from libmproxy/protocol2/transparent_proxy.py)6
-rw-r--r--libmproxy/proxy/primitives.py15
-rw-r--r--libmproxy/proxy/root_context.py (renamed from libmproxy/protocol2/root_context.py)14
-rw-r--r--libmproxy/proxy/server.py23
25 files changed, 816 insertions, 794 deletions
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py
index 1940e390..c25f7267 100644
--- a/libmproxy/console/common.py
+++ b/libmproxy/console/common.py
@@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING
import netlib.utils
from .. import utils
-from ..protocol.http import decoded
+from ..models import decoded
from . import signals
diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py
index 1e0f0c17..8b828653 100644
--- a/libmproxy/console/flowview.py
+++ b/libmproxy/console/flowview.py
@@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING
from . import common, grideditor, contentview, signals, searchable, tabs
from . import flowdetailview
from .. import utils, controller
-from ..protocol.http import HTTPRequest, HTTPResponse, decoded
+from ..models import HTTPRequest, HTTPResponse, decoded
class SearchError(Exception):
diff --git a/libmproxy/filt.py b/libmproxy/filt.py
index 25747bc6..cfd3a1bc 100644
--- a/libmproxy/filt.py
+++ b/libmproxy/filt.py
@@ -35,7 +35,7 @@ from __future__ import absolute_import
import re
import sys
import pyparsing as pp
-from .protocol.http import decoded
+from .models import decoded
class _Token:
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index a2f57512..00ec83d2 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -8,19 +8,18 @@ import Cookie
import cookielib
import os
import re
-from libmproxy.protocol.http import HTTPFlow
-from libmproxy.protocol2.http_replay import RequestReplayThread
+import urlparse
+
from netlib import odict, wsgi
from netlib.http.semantics import CONTENT_MISSING
import netlib.http
-
-from . import controller, protocol, tnetstring, filt, script, version
+from . import controller, tnetstring, filt, script, version
from .onboarding import app
-from .protocol import http
from .proxy.config import HostMatcher
-from .proxy.connection import ClientConnection, ServerConnection
-import urlparse
+from .protocol.http_replay import RequestReplayThread
+from .protocol import Kill
+from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest
class AppRegistry:
@@ -790,7 +789,7 @@ class FlowMaster(controller.Master):
rflow = self.server_playback.next_flow(flow)
if not rflow:
return None
- response = http.HTTPResponse.from_state(rflow.response.get_state())
+ response = HTTPResponse.from_state(rflow.response.get_state())
response.is_replay = True
if self.refresh_server_playback:
response.refresh()
@@ -836,10 +835,10 @@ class FlowMaster(controller.Master):
sni=host,
ssl_established=True
))
- f = http.HTTPFlow(c, s)
+ f = HTTPFlow(c, s)
headers = odict.ODictCaseless()
- req = http.HTTPRequest(
+ req = HTTPRequest(
"absolute",
method,
scheme,
@@ -981,7 +980,7 @@ class FlowMaster(controller.Master):
)
if err:
self.add_event("Error in wsgi app. %s" % err, "error")
- f.reply(protocol.KILL)
+ f.reply(Kill)
return
if f not in self.state.flows: # don't add again on replay
self.state.add_flow(f)
@@ -998,7 +997,7 @@ class FlowMaster(controller.Master):
if self.stream_large_bodies:
self.stream_large_bodies.run(f, False)
except netlib.http.HttpError:
- f.reply(protocol.KILL)
+ f.reply(Kill)
return
f.reply()
diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py
new file mode 100644
index 00000000..3947847c
--- /dev/null
+++ b/libmproxy/models/__init__.py
@@ -0,0 +1,16 @@
+from __future__ import (absolute_import, print_function, division)
+
+from .http import (
+ HTTPFlow, HTTPRequest, HTTPResponse, decoded,
+ make_error_response, make_connect_request, make_connect_response
+)
+from .connections import ClientConnection, ServerConnection
+from .flow import Flow, Error
+
+__all__ = [
+ "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded"
+ "make_error_response", "make_connect_request",
+ "make_connect_response",
+ "ClientConnection", "ServerConnection",
+ "Flow", "Error",
+]
diff --git a/libmproxy/proxy/connection.py b/libmproxy/models/connections.py
index 94f318f6..98bae3cc 100644
--- a/libmproxy/proxy/connection.py
+++ b/libmproxy/models/connections.py
@@ -190,4 +190,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
tcp.TCPClient.finish(self)
self.timestamp_end = utils.timestamp()
+
ServerConnection._stateobject_attributes["via"] = ServerConnection
diff --git a/libmproxy/protocol/primitives.py b/libmproxy/models/flow.py
index c663f0c5..58287e5b 100644
--- a/libmproxy/protocol/primitives.py
+++ b/libmproxy/models/flow.py
@@ -3,9 +3,7 @@ import copy
import uuid
from .. import stateobject, utils, version
-from ..proxy.connection import ClientConnection, ServerConnection
-
-KILL = 0 # const for killed requests
+from .connections import ClientConnection, ServerConnection
class Error(stateobject.StateObject):
@@ -140,9 +138,11 @@ class Flow(stateobject.StateObject):
"""
Kill this request.
"""
+ from ..protocol import Kill
+
self.error = Error("Connection killed")
self.intercepted = False
- self.reply(KILL)
+ self.reply(Kill)
master.handle_error(self)
def intercept(self, master):
diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/models/http.py
index a26ddbb4..fb2f305b 100644
--- a/libmproxy/protocol/http_wrappers.py
+++ b/libmproxy/models/http.py
@@ -1,39 +1,16 @@
-from __future__ import absolute_import
+from __future__ import (absolute_import, print_function, division)
import Cookie
import copy
-import time
from email.utils import parsedate_tz, formatdate, mktime_tz
+import time
+from libmproxy import utils
from netlib import odict, encoding
-from netlib.http import semantics, CONTENT_MISSING
-from .. import utils, stateobject
-
-
-class decoded(object):
- """
- A context manager that decodes a request or response, and then
- re-encodes it with the same encoding after execution of the block.
-
- Example:
- with decoded(request):
- request.content = request.content.replace("foo", "bar")
- """
-
- def __init__(self, o):
- self.o = o
- ce = o.headers.get_first("content-encoding")
- if ce in encoding.ENCODINGS:
- self.ce = ce
- else:
- self.ce = None
-
- def __enter__(self):
- if self.ce:
- self.o.decode()
-
- def __exit__(self, type, value, tb):
- if self.ce:
- self.o.encode(self.ce)
+from netlib.http import status_codes
+from netlib.tcp import Address
+from netlib.http.semantics import Request, Response, CONTENT_MISSING
+from .. import version, stateobject
+from .flow import Flow
class MessageMixin(stateobject.StateObject):
@@ -116,7 +93,7 @@ class MessageMixin(stateobject.StateObject):
return c
-class HTTPRequest(MessageMixin, semantics.Request):
+class HTTPRequest(MessageMixin, Request):
"""
An HTTP request.
@@ -176,7 +153,7 @@ class HTTPRequest(MessageMixin, semantics.Request):
timestamp_end=None,
form_out=None,
):
- semantics.Request.__init__(
+ Request.__init__(
self,
form_in,
method,
@@ -277,7 +254,7 @@ class HTTPRequest(MessageMixin, semantics.Request):
return c
-class HTTPResponse(MessageMixin, semantics.Response):
+class HTTPResponse(MessageMixin, Response):
"""
An HTTP response.
@@ -310,7 +287,7 @@ class HTTPResponse(MessageMixin, semantics.Response):
timestamp_start=None,
timestamp_end=None,
):
- semantics.Response.__init__(
+ Response.__init__(
self,
httpversion,
status_code,
@@ -411,3 +388,167 @@ class HTTPResponse(MessageMixin, semantics.Response):
c.append(self._refresh_cookie(i, delta))
if c:
self.headers["set-cookie"] = c
+
+
+class HTTPFlow(Flow):
+ """
+ A HTTPFlow is a collection of objects representing a single HTTP
+ transaction. The main attributes are:
+
+ request: HTTPRequest object
+ response: HTTPResponse object
+ error: Error object
+ server_conn: ServerConnection object
+ client_conn: ClientConnection object
+
+ Note that it's possible for a Flow to have both a response and an error
+ object. This might happen, for instance, when a response was received
+ from the server, but there was an error sending it back to the client.
+
+ The following additional attributes are exposed:
+
+ intercepted: Is this flow currently being intercepted?
+ live: Does this flow have a live client connection?
+ """
+
+ def __init__(self, client_conn, server_conn, live=None):
+ super(HTTPFlow, self).__init__("http", client_conn, server_conn, live)
+ self.request = None
+ """@type: HTTPRequest"""
+ self.response = None
+ """@type: HTTPResponse"""
+
+ _stateobject_attributes = Flow._stateobject_attributes.copy()
+ _stateobject_attributes.update(
+ request=HTTPRequest,
+ response=HTTPResponse
+ )
+
+ @classmethod
+ def from_state(cls, state):
+ f = cls(None, None)
+ f.load_state(state)
+ return f
+
+ def __repr__(self):
+ s = "<HTTPFlow"
+ for a in ("request", "response", "error", "client_conn", "server_conn"):
+ if getattr(self, a, False):
+ s += "\r\n %s = {flow.%s}" % (a, a)
+ s += ">"
+ return s.format(flow=self)
+
+ def copy(self):
+ f = super(HTTPFlow, self).copy()
+ if self.request:
+ f.request = self.request.copy()
+ if self.response:
+ f.response = self.response.copy()
+ return f
+
+ def match(self, f):
+ """
+ Match this flow against a compiled filter expression. Returns True
+ if matched, False if not.
+
+ If f is a string, it will be compiled as a filter expression. If
+ the expression is invalid, ValueError is raised.
+ """
+ if isinstance(f, basestring):
+ from .. import filt
+
+ f = filt.parse(f)
+ if not f:
+ raise ValueError("Invalid filter expression.")
+ if f:
+ return f(self)
+ return True
+
+ def replace(self, pattern, repl, *args, **kwargs):
+ """
+ Replaces a regular expression pattern with repl in both request and
+ response of the flow. Encoded content will be decoded before
+ replacement, and re-encoded afterwards.
+
+ Returns the number of replacements made.
+ """
+ c = self.request.replace(pattern, repl, *args, **kwargs)
+ if self.response:
+ c += self.response.replace(pattern, repl, *args, **kwargs)
+ return c
+
+
+class decoded(object):
+ """
+ A context manager that decodes a request or response, and then
+ re-encodes it with the same encoding after execution of the block.
+
+ Example:
+ with decoded(request):
+ request.content = request.content.replace("foo", "bar")
+ """
+
+ def __init__(self, o):
+ self.o = o
+ ce = o.headers.get_first("content-encoding")
+ if ce in encoding.ENCODINGS:
+ self.ce = ce
+ else:
+ self.ce = None
+
+ def __enter__(self):
+ if self.ce:
+ self.o.decode()
+
+ def __exit__(self, type, value, tb):
+ if self.ce:
+ self.o.encode(self.ce)
+
+
+def make_error_response(status_code, message, headers=None):
+ response = status_codes.RESPONSES.get(status_code, "Unknown")
+ body = """
+ <html>
+ <head>
+ <title>%d %s</title>
+ </head>
+ <body>%s</body>
+ </html>
+ """.strip() % (status_code, response, message)
+
+ if not headers:
+ headers = odict.ODictCaseless()
+ headers["Server"] = [version.NAMEVERSION]
+ headers["Connection"] = ["close"]
+ headers["Content-Length"] = [len(body)]
+ headers["Content-Type"] = ["text/html"]
+
+ return HTTPResponse(
+ (1, 1), # FIXME: Should be a string.
+ status_code,
+ response,
+ headers,
+ body,
+ )
+
+
+def make_connect_request(address):
+ address = Address.wrap(address)
+ return HTTPRequest(
+ "authority", "CONNECT", None, address.host, address.port, None, (1, 1),
+ odict.ODictCaseless(), ""
+ )
+
+
+def make_connect_response(httpversion):
+ headers = odict.ODictCaseless([
+ ["Content-Length", "0"],
+ ["Proxy-Agent", version.NAMEVERSION]
+ ])
+ return HTTPResponse(
+ httpversion,
+ 200,
+ "Connection established",
+ headers,
+ "",
+ )
diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py
index bbc20dba..c582592b 100644
--- a/libmproxy/protocol/__init__.py
+++ b/libmproxy/protocol/__init__.py
@@ -1 +1,12 @@
-from .primitives import *
+from __future__ import (absolute_import, print_function, division)
+from .base import Layer, ServerConnectionMixin, Log, Kill
+from .http import Http1Layer, Http2Layer
+from .tls import TlsLayer, is_tls_record_magic
+from .rawtcp import RawTCPLayer
+
+__all__ = [
+ "Layer", "ServerConnectionMixin", "Log", "Kill",
+ "Http1Layer", "Http2Layer",
+ "TlsLayer", "is_tls_record_magic",
+ "RawTCPLayer"
+]
diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol/base.py
index 2b47cc26..d22a71c6 100644
--- a/libmproxy/protocol2/layer.py
+++ b/libmproxy/protocol/base.py
@@ -2,22 +2,25 @@
mitmproxy protocol architecture
In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other.
-For example, the following scenarios depict possible scenarios (lowest layer first):
+For example, the following scenarios depict possible settings (lowest layer first):
Transparent HTTP proxy, no SSL:
- TransparentModeLayer
+ TransparentProxy
+ Http1Layer
HttpLayer
Regular proxy, CONNECT request with WebSockets over SSL:
- RegularModeLayer
+ HttpProxy
+ Http1Layer
HttpLayer
SslLayer
WebsocketLayer (or TcpLayer)
Automated protocol detection by peeking into the buffer:
- TransparentModeLayer
- SslLayer
+ TransparentProxy
+ TLSLayer
Http2Layer
+ HttpLayer
Communication between layers is done as follows:
- lower layers provide context information to higher layers
@@ -31,8 +34,7 @@ Further goals:
"""
from __future__ import (absolute_import, print_function, division)
from netlib import tcp
-from ..proxy import Log
-from ..proxy.connection import ServerConnection
+from ..models import ServerConnection
from ..exceptions import ProtocolException
@@ -136,3 +138,15 @@ class ServerConnectionMixin(object):
except tcp.NetLibError as e:
raise ProtocolException(
"Server connection to '%s' failed: %s" % (self.server_conn.address, e), e)
+
+
+class Log(object):
+ def __init__(self, msg, level="info"):
+ self.msg = msg
+ self.level = level
+
+
+class Kill(Exception):
+ """
+ Kill a connection.
+ """
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py
index bde7b088..fc57f6df 100644
--- a/libmproxy/protocol/http.py
+++ b/libmproxy/protocol/http.py
@@ -1,92 +1,538 @@
-from __future__ import absolute_import
+from __future__ import (absolute_import, print_function, division)
-from .primitives import Flow
+from netlib import tcp
+from netlib.http import http1, HttpErrorConnClosed, HttpError
+from netlib.http.semantics import CONTENT_MISSING
+from netlib import odict
+from netlib.tcp import NetLibError, Address
+from netlib.http.http1 import HTTP1Protocol
+from netlib.http.http2 import HTTP2Protocol
-from .http_wrappers import decoded, HTTPRequest, HTTPResponse
+from .. import utils
+from ..exceptions import InvalidCredentials, HttpException, ProtocolException
+from ..models import (
+ HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error
+)
+from .base import Layer, Kill
-class HTTPFlow(Flow):
- """
- A HTTPFlow is a collection of objects representing a single HTTP
- transaction. The main attributes are:
- request: HTTPRequest object
- response: HTTPResponse object
- error: Error object
- server_conn: ServerConnection object
- client_conn: ClientConnection object
+class _HttpLayer(Layer):
+ supports_streaming = False
+
+ def read_request(self):
+ raise NotImplementedError()
+
+ def send_request(self, request):
+ raise NotImplementedError()
+
+ def read_response(self, request_method):
+ raise NotImplementedError()
+
+ def send_response(self, response):
+ raise NotImplementedError()
+
+
+class _StreamingHttpLayer(_HttpLayer):
+ supports_streaming = True
+
+ def read_response_headers(self):
+ raise NotImplementedError
+
+ def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
+ raise NotImplementedError()
+ yield "this is a generator"
+
+ def send_response_headers(self, response):
+ raise NotImplementedError
+
+ def send_response_body(self, response, chunks):
+ raise NotImplementedError()
+
+
+class Http1Layer(_StreamingHttpLayer):
+ def __init__(self, ctx, mode):
+ super(Http1Layer, self).__init__(ctx)
+ self.mode = mode
+ self.client_protocol = HTTP1Protocol(self.client_conn)
+ self.server_protocol = HTTP1Protocol(self.server_conn)
+
+ def read_request(self):
+ return HTTPRequest.from_protocol(
+ self.client_protocol,
+ body_size_limit=self.config.body_size_limit
+ )
+
+ def send_request(self, request):
+ self.server_conn.send(self.server_protocol.assemble(request))
+
+ def read_response(self, request_method):
+ return HTTPResponse.from_protocol(
+ self.server_protocol,
+ request_method=request_method,
+ body_size_limit=self.config.body_size_limit,
+ include_body=True
+ )
+
+ def send_response(self, response):
+ self.client_conn.send(self.client_protocol.assemble(response))
+
+ def read_response_headers(self):
+ return HTTPResponse.from_protocol(
+ self.server_protocol,
+ request_method=None, # does not matter if we don't read the body.
+ body_size_limit=self.config.body_size_limit,
+ include_body=False
+ )
+
+ def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
+ return self.server_protocol.read_http_body_chunked(
+ headers,
+ self.config.body_size_limit,
+ request_method,
+ response_code,
+ False,
+ max_chunk_size
+ )
+
+ def send_response_headers(self, response):
+ h = self.client_protocol._assemble_response_first_line(response)
+ self.client_conn.wfile.write(h + "\r\n")
+ h = self.client_protocol._assemble_response_headers(
+ response,
+ preserve_transfer_encoding=True
+ )
+ self.client_conn.send(h + "\r\n")
+
+ def send_response_body(self, response, chunks):
+ if self.client_protocol.has_chunked_encoding(response.headers):
+ chunks = (
+ "%d\r\n%s\r\n" % (len(chunk), chunk)
+ for chunk in chunks
+ )
+ for chunk in chunks:
+ self.client_conn.send(chunk)
- Note that it's possible for a Flow to have both a response and an error
- object. This might happen, for instance, when a response was received
- from the server, but there was an error sending it back to the client.
+ def connect(self):
+ self.ctx.connect()
+ self.server_protocol = HTTP1Protocol(self.server_conn)
- The following additional attributes are exposed:
+ def reconnect(self):
+ self.ctx.reconnect()
+ self.server_protocol = HTTP1Protocol(self.server_conn)
- intercepted: Is this flow currently being intercepted?
- live: Does this flow have a live client connection?
+ def set_server(self, *args, **kwargs):
+ self.ctx.set_server(*args, **kwargs)
+ self.server_protocol = HTTP1Protocol(self.server_conn)
+
+ def __call__(self):
+ layer = HttpLayer(self, self.mode)
+ layer()
+
+
+# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
+class Http2Layer(_HttpLayer):
+ def __init__(self, ctx, mode):
+ super(Http2Layer, self).__init__(ctx)
+ self.mode = mode
+ self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True,
+ unhandled_frame_cb=self.handle_unexpected_frame)
+ self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
+ unhandled_frame_cb=self.handle_unexpected_frame)
+
+ def read_request(self):
+ request = HTTPRequest.from_protocol(
+ self.client_protocol,
+ body_size_limit=self.config.body_size_limit
+ )
+ self._stream_id = request.stream_id
+ return request
+
+ def send_request(self, message):
+ # TODO: implement flow control and WINDOW_UPDATE frames
+ self.server_conn.send(self.server_protocol.assemble(message))
+
+ def read_response(self, request_method):
+ return HTTPResponse.from_protocol(
+ self.server_protocol,
+ request_method=request_method,
+ body_size_limit=self.config.body_size_limit,
+ include_body=True,
+ stream_id=self._stream_id
+ )
+
+ def send_response(self, message):
+ # TODO: implement flow control and WINDOW_UPDATE frames
+ self.client_conn.send(self.client_protocol.assemble(message))
+
+ def connect(self):
+ self.ctx.connect()
+ self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
+ unhandled_frame_cb=self.handle_unexpected_frame)
+ self.server_protocol.perform_connection_preface()
+
+ def reconnect(self):
+ self.ctx.reconnect()
+ self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
+ unhandled_frame_cb=self.handle_unexpected_frame)
+ self.server_protocol.perform_connection_preface()
+
+ def set_server(self, *args, **kwargs):
+ self.ctx.set_server(*args, **kwargs)
+ self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
+ unhandled_frame_cb=self.handle_unexpected_frame)
+ self.server_protocol.perform_connection_preface()
+
+ def __call__(self):
+ self.server_protocol.perform_connection_preface()
+ layer = HttpLayer(self, self.mode)
+ layer()
+
+ def handle_unexpected_frame(self, frm):
+ self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info")
+
+
+class ConnectServerConnection(object):
+ """
+ "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
"""
- def __init__(self, client_conn, server_conn, live=None):
- super(HTTPFlow, self).__init__("http", client_conn, server_conn, live)
- self.request = None
- """@type: HTTPRequest"""
- self.response = None
- """@type: HTTPResponse"""
-
- _stateobject_attributes = Flow._stateobject_attributes.copy()
- _stateobject_attributes.update(
- request=HTTPRequest,
- response=HTTPResponse
- )
-
- @classmethod
- def from_state(cls, state):
- f = cls(None, None)
- f.load_state(state)
- return f
-
- def __repr__(self):
- s = "<HTTPFlow"
- for a in ("request", "response", "error", "client_conn", "server_conn"):
- if getattr(self, a, False):
- s += "\r\n %s = {flow.%s}" % (a, a)
- s += ">"
- return s.format(flow=self)
-
- def copy(self):
- f = super(HTTPFlow, self).copy()
- if self.request:
- f.request = self.request.copy()
- if self.response:
- f.response = self.response.copy()
- return f
-
- def match(self, f):
- """
- Match this flow against a compiled filter expression. Returns True
- if matched, False if not.
+ def __init__(self, address, ctx):
+ self.address = tcp.Address.wrap(address)
+ self._ctx = ctx
- If f is a string, it will be compiled as a filter expression. If
- the expression is invalid, ValueError is raised.
- """
- if isinstance(f, basestring):
- from .. import filt
+ @property
+ def via(self):
+ return self._ctx.server_conn
- f = filt.parse(f)
- if not f:
- raise ValueError("Invalid filter expression.")
- if f:
- return f(self)
- return True
+ def __getattr__(self, item):
+ return getattr(self.via, item)
- def replace(self, pattern, repl, *args, **kwargs):
- """
- Replaces a regular expression pattern with repl in both request and
- response of the flow. Encoded content will be decoded before
- replacement, and re-encoded afterwards.
- Returns the number of replacements made.
+class UpstreamConnectLayer(Layer):
+ def __init__(self, ctx, connect_request):
+ super(UpstreamConnectLayer, self).__init__(ctx)
+ self.connect_request = connect_request
+ self.server_conn = ConnectServerConnection(
+ (connect_request.host, connect_request.port),
+ self.ctx
+ )
+
+ def __call__(self):
+ layer = self.ctx.next_layer(self)
+ layer()
+
+ def connect(self):
+ if not self.server_conn:
+ self.ctx.connect()
+ self.send_request(self.connect_request)
+ else:
+ pass # swallow the message
+
+ def reconnect(self):
+ self.ctx.reconnect()
+ self.send_request(self.connect_request)
+ resp = self.read_response("CONNECT")
+ if resp.code != 200:
+ raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
+
+ def set_server(self, address, server_tls=None, sni=None, depth=1):
+ if depth == 1:
+ if self.ctx.server_conn:
+ self.ctx.reconnect()
+ address = Address.wrap(address)
+ self.connect_request.host = address.host
+ self.connect_request.port = address.port
+ self.server_conn.address = address
+ else:
+ self.ctx.set_server(address, server_tls, sni, depth - 1)
+
+
+class HttpLayer(Layer):
+ def __init__(self, ctx, mode):
+ super(HttpLayer, self).__init__(ctx)
+ self.mode = mode
+ self.__original_server_conn = None
+ "Contains the original destination in transparent mode, which needs to be restored"
+ "if an inline script modified the target server for a single http request"
+
+ def __call__(self):
+ if self.mode == "transparent":
+ self.__original_server_conn = self.server_conn
+ while True:
+ try:
+ flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
+
+ try:
+ request = self.read_request()
+ except tcp.NetLibError:
+ # don't throw an error for disconnects that happen
+ # before/between requests.
+ return
+
+ self.log("request", "debug", [repr(request)])
+
+ # Handle Proxy Authentication
+ self.authenticate(request)
+
+ # Regular Proxy Mode: Handle CONNECT
+ if self.mode == "regular" and request.form_in == "authority":
+ self.handle_regular_mode_connect(request)
+ return
+
+ # Make sure that the incoming request matches our expectations
+ self.validate_request(request)
+
+ flow.request = request
+ self.process_request_hook(flow)
+
+ if not flow.response:
+ self.establish_server_connection(flow)
+ self.get_response_from_server(flow)
+
+ self.send_response_to_client(flow)
+
+ if self.check_close_connection(flow):
+ return
+
+ # TODO: Implement HTTP Upgrade
+
+ # Upstream Proxy Mode: Handle CONNECT
+ if flow.request.form_in == "authority" and flow.response.code == 200:
+ self.handle_upstream_mode_connect(flow.request.copy())
+ return
+
+ except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
+ if flow.request and not flow.response:
+ flow.error = Error(repr(e))
+ self.channel.ask("error", flow)
+ try:
+ self.send_response(make_error_response(
+ getattr(e, "code", 502),
+ repr(e)
+ ))
+ except NetLibError:
+ pass
+ if isinstance(e, ProtocolException):
+ raise e
+ else:
+ raise ProtocolException("Error in HTTP connection: %s" % repr(e), e)
+ finally:
+ flow.live = False
+
+ def handle_regular_mode_connect(self, request):
+ self.set_server((request.host, request.port))
+ self.send_response(make_connect_response(request.httpversion))
+ layer = self.ctx.next_layer(self)
+ layer()
+
+ def handle_upstream_mode_connect(self, connect_request):
+ layer = UpstreamConnectLayer(self, connect_request)
+ layer()
+
+ def check_close_connection(self, flow):
"""
- c = self.request.replace(pattern, repl, *args, **kwargs)
- if self.response:
- c += self.response.replace(pattern, repl, *args, **kwargs)
- return c
+ Checks if the connection should be closed depending on the HTTP
+ semantics. Returns True, if so.
+ """
+
+ # TODO: add logic for HTTP/2
+
+ close_connection = (
+ http1.HTTP1Protocol.connection_close(
+ flow.request.httpversion,
+ flow.request.headers
+ ) or http1.HTTP1Protocol.connection_close(
+ flow.response.httpversion,
+ flow.response.headers
+ ) or http1.HTTP1Protocol.expected_http_body_size(
+ flow.response.headers,
+ False,
+ flow.request.method,
+ flow.response.code) == -1
+ )
+ if flow.request.form_in == "authority" and flow.response.code == 200:
+ # Workaround for
+ # https://github.com/mitmproxy/mitmproxy/issues/313: Some
+ # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
+ # and no Content-Length header
+
+ return False
+ return close_connection
+
+ def send_response_to_client(self, flow):
+ if not (self.supports_streaming and flow.response.stream):
+ # no streaming:
+ # we already received the full response from the server and can
+ # send it to the client straight away.
+ self.send_response(flow.response)
+ else:
+ # streaming:
+ # First send the headers and then transfer the response incrementally
+ self.send_response_headers(flow.response)
+ chunks = self.read_response_body(
+ flow.response.headers,
+ flow.request.method,
+ flow.response.code,
+ max_chunk_size=4096
+ )
+ if callable(flow.response.stream):
+ chunks = flow.response.stream(chunks)
+ self.send_response_body(flow.response, chunks)
+ flow.response.timestamp_end = utils.timestamp()
+
+ def get_response_from_server(self, flow):
+ def get_response():
+ self.send_request(flow.request)
+ if self.supports_streaming:
+ flow.response = self.read_response_headers()
+ else:
+ flow.response = self.read_response()
+
+ try:
+ get_response()
+ except (tcp.NetLibError, HttpErrorConnClosed) as v:
+ self.log(
+ "server communication error: %s" % repr(v),
+ level="debug"
+ )
+ # In any case, we try to reconnect at least once. This is
+ # necessary because it might be possible that we already
+ # initiated an upstream connection after clientconnect that
+ # has already been expired, e.g consider the following event
+ # log:
+ # > clientconnect (transparent mode destination known)
+ # > serverconnect (required for client tls handshake)
+ # > read n% of large request
+ # > server detects timeout, disconnects
+ # > read (100-n)% of large request
+ # > send large request upstream
+ self.reconnect()
+ get_response()
+
+ # call the appropriate script hook - this is an opportunity for an
+ # inline script to set flow.stream = True
+ flow = self.channel.ask("responseheaders", flow)
+ if flow is None or flow == Kill:
+ raise Kill()
+
+ if self.supports_streaming:
+ if flow.response.stream:
+ flow.response.content = CONTENT_MISSING
+ else:
+ flow.response.content = "".join(self.read_response_body(
+ flow.response.headers,
+ flow.request.method,
+ flow.response.code
+ ))
+ flow.response.timestamp_end = utils.timestamp()
+
+ # no further manipulation of self.server_conn beyond this point
+ # we can safely set it as the final attribute value here.
+ flow.server_conn = self.server_conn
+
+ self.log(
+ "response",
+ "debug",
+ [repr(flow.response)]
+ )
+ response_reply = self.channel.ask("response", flow)
+ if response_reply is None or response_reply == Kill:
+ raise Kill()
+
+ def process_request_hook(self, flow):
+ # Determine .scheme, .host and .port attributes for inline scripts.
+ # For absolute-form requests, they are directly given in the request.
+ # For authority-form requests, we only need to determine the request scheme.
+ # For relative-form requests, we need to determine host and port as
+ # well.
+ if self.mode == "regular":
+ pass # only absolute-form at this point, nothing to do here.
+ elif self.mode == "upstream":
+ if flow.request.form_in == "authority":
+ flow.request.scheme = "http" # pseudo value
+ else:
+ flow.request.host = self.__original_server_conn.address.host
+ flow.request.port = self.__original_server_conn.address.port
+ flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http"
+
+ request_reply = self.channel.ask("request", flow)
+ if request_reply is None or request_reply == Kill:
+ raise Kill()
+ if isinstance(request_reply, HTTPResponse):
+ flow.response = request_reply
+ return
+
+ def establish_server_connection(self, flow):
+ address = tcp.Address((flow.request.host, flow.request.port))
+ tls = (flow.request.scheme == "https")
+
+ if self.mode == "regular" or self.mode == "transparent":
+ # If there's an existing connection that doesn't match our expectations, kill it.
+ if address != self.server_conn.address or tls != self.server_conn.ssl_established:
+ self.set_server(address, tls, address.host)
+ # Establish connection is neccessary.
+ if not self.server_conn:
+ self.connect()
+
+ # SetServer is not guaranteed to work with TLS:
+ # If there's not TlsLayer below which could catch the exception,
+ # TLS will not be established.
+ if tls and not self.server_conn.tls_established:
+ raise ProtocolException(
+ "Cannot upgrade to SSL, no TLS layer on the protocol stack.")
+ else:
+ if not self.server_conn:
+ self.connect()
+ if tls:
+ raise HttpException("Cannot change scheme in upstream proxy mode.")
+ """
+ # This is a very ugly (untested) workaround to solve a very ugly problem.
+ if self.server_conn and self.server_conn.tls_established and not ssl:
+ self.reconnect()
+ elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
+ if self.server_conn.tls_established:
+ self.reconnect()
+
+ self.send_request(make_connect_request(address))
+ tls_layer = TlsLayer(self, False, True)
+ tls_layer._establish_tls_with_server()
+ """
+
+ def validate_request(self, request):
+ if request.form_in == "absolute" and request.scheme != "http":
+ self.send_response(
+ make_error_response(400, "Invalid request scheme: %s" % request.scheme))
+ raise HttpException("Invalid request scheme: %s" % request.scheme)
+
+ expected_request_forms = {
+ "regular": ("absolute",), # an authority request would already be handled.
+ "upstream": ("authority", "absolute"),
+ "transparent": ("relative",)
+ }
+
+ allowed_request_forms = expected_request_forms[self.mode]
+ if request.form_in not in allowed_request_forms:
+ err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
+ " or ".join(allowed_request_forms), request.form_in
+ )
+ self.send_response(make_error_response(400, err_message))
+ raise HttpException(err_message)
+
+ if self.mode == "regular":
+ request.form_out = "relative"
+
+ def authenticate(self, request):
+ if self.config.authenticator:
+ if self.config.authenticator.authenticate(request.headers):
+ self.config.authenticator.clean(request.headers)
+ else:
+ self.send_response(make_error_response(
+ 407,
+ "Proxy Authentication Required",
+ odict.ODictCaseless(
+ [
+ [k, v] for k, v in
+ self.config.authenticator.auth_challenge_headers().items()
+ ])
+ ))
+ raise InvalidCredentials("Proxy Authentication Required")
diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol/http_replay.py
index 872ef9cd..e0144c93 100644
--- a/libmproxy/protocol2/http_replay.py
+++ b/libmproxy/protocol/http_replay.py
@@ -1,14 +1,14 @@
import threading
+
from netlib.http import HttpError
from netlib.http.http1 import HTTP1Protocol
from netlib.tcp import NetLibError
-
from ..controller import Channel
-from ..protocol import KILL, Error
-from ..protocol.http_wrappers import HTTPResponse
-from ..proxy import Log, Kill
-from ..proxy.connection import ServerConnection
-from .http import make_connect_request
+from ..models import Error, HTTPResponse, ServerConnection, make_connect_request
+from .base import Log, Kill
+
+
+# TODO: Doesn't really belong into libmproxy.protocol...
class RequestReplayThread(threading.Thread):
@@ -35,7 +35,7 @@ class RequestReplayThread(threading.Thread):
# If we have a channel, run script hooks.
if self.channel:
request_reply = self.channel.ask("request", self.flow)
- if request_reply is None or request_reply == KILL:
+ if request_reply is None or request_reply == Kill:
raise Kill()
elif isinstance(request_reply, HTTPResponse):
self.flow.response = request_reply
@@ -81,7 +81,7 @@ class RequestReplayThread(threading.Thread):
)
if self.channel:
response_reply = self.channel.ask("response", self.flow)
- if response_reply is None or response_reply == KILL:
+ if response_reply is None or response_reply == Kill:
raise Kill()
except (HttpError, NetLibError) as v:
self.flow.error = Error(repr(v))
diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol/rawtcp.py
index b10217f1..86468773 100644
--- a/libmproxy/protocol2/rawtcp.py
+++ b/libmproxy/protocol/rawtcp.py
@@ -7,15 +7,15 @@ from OpenSSL import SSL
from netlib.tcp import NetLibError
from netlib.utils import cleanBin
from ..exceptions import ProtocolException
-from .layer import Layer
+from .base import Layer
-class RawTcpLayer(Layer):
+class RawTCPLayer(Layer):
chunk_size = 4096
def __init__(self, ctx, logging=True):
self.logging = logging
- super(RawTcpLayer, self).__init__(ctx)
+ super(RawTCPLayer, self).__init__(ctx)
def __call__(self):
self.connect()
diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol/tls.py
index 73bb12f3..b85a6595 100644
--- a/libmproxy/protocol2/tls.py
+++ b/libmproxy/protocol/tls.py
@@ -8,7 +8,7 @@ from netlib.tcp import NetLibError, NetLibInvalidCertificateError
from netlib.http.http1 import HTTP1Protocol
from ..contrib.tls._constructs import ClientHello
from ..exceptions import ProtocolException
-from .layer import Layer
+from .base import Layer
def is_tls_record_magic(d):
diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py
deleted file mode 100644
index a508ae8b..00000000
--- a/libmproxy/protocol2/http.py
+++ /dev/null
@@ -1,588 +0,0 @@
-from __future__ import (absolute_import, print_function, division)
-
-from netlib import tcp
-from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError
-from netlib.http.semantics import CONTENT_MISSING
-from netlib import odict
-from netlib.tcp import NetLibError, Address
-from netlib.http.http1 import HTTP1Protocol
-from netlib.http.http2 import HTTP2Protocol
-
-from .. import version, utils
-from ..exceptions import InvalidCredentials, HttpException, ProtocolException
-from .layer import Layer
-from ..proxy import Kill
-from libmproxy.protocol import KILL, Error
-from libmproxy.protocol.http import HTTPFlow
-from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest
-
-
-class _HttpLayer(Layer):
- supports_streaming = False
-
- def read_request(self):
- raise NotImplementedError()
-
- def send_request(self, request):
- raise NotImplementedError()
-
- def read_response(self, request_method):
- raise NotImplementedError()
-
- def send_response(self, response):
- raise NotImplementedError()
-
-
-class _StreamingHttpLayer(_HttpLayer):
- supports_streaming = True
-
- def read_response_headers(self):
- raise NotImplementedError
-
- def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
- raise NotImplementedError()
- yield "this is a generator"
-
- def send_response_headers(self, response):
- raise NotImplementedError
-
- def send_response_body(self, response, chunks):
- raise NotImplementedError()
-
-
-class Http1Layer(_StreamingHttpLayer):
- def __init__(self, ctx, mode):
- super(Http1Layer, self).__init__(ctx)
- self.mode = mode
- self.client_protocol = HTTP1Protocol(self.client_conn)
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def read_request(self):
- return HTTPRequest.from_protocol(
- self.client_protocol,
- body_size_limit=self.config.body_size_limit
- )
-
- def send_request(self, request):
- self.server_conn.send(self.server_protocol.assemble(request))
-
- def read_response(self, request_method):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=request_method,
- body_size_limit=self.config.body_size_limit,
- include_body=True
- )
-
- def send_response(self, response):
- self.client_conn.send(self.client_protocol.assemble(response))
-
- def read_response_headers(self):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=None, # does not matter if we don't read the body.
- body_size_limit=self.config.body_size_limit,
- include_body=False
- )
-
- def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
- return self.server_protocol.read_http_body_chunked(
- headers,
- self.config.body_size_limit,
- request_method,
- response_code,
- False,
- max_chunk_size
- )
-
- def send_response_headers(self, response):
- h = self.client_protocol._assemble_response_first_line(response)
- self.client_conn.wfile.write(h + "\r\n")
- h = self.client_protocol._assemble_response_headers(
- response,
- preserve_transfer_encoding=True
- )
- self.client_conn.send(h + "\r\n")
-
- def send_response_body(self, response, chunks):
- if self.client_protocol.has_chunked_encoding(response.headers):
- chunks = (
- "%d\r\n%s\r\n" % (len(chunk), chunk)
- for chunk in chunks
- )
- for chunk in chunks:
- self.client_conn.send(chunk)
-
- def connect(self):
- self.ctx.connect()
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def reconnect(self):
- self.ctx.reconnect()
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def set_server(self, *args, **kwargs):
- self.ctx.set_server(*args, **kwargs)
- self.server_protocol = HTTP1Protocol(self.server_conn)
-
- def __call__(self):
- layer = HttpLayer(self, self.mode)
- layer()
-
-
-# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
-class Http2Layer(_HttpLayer):
- def __init__(self, ctx, mode):
- super(Http2Layer, self).__init__(ctx)
- self.mode = mode
- self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
-
- def read_request(self):
- request = HTTPRequest.from_protocol(
- self.client_protocol,
- body_size_limit=self.config.body_size_limit
- )
- self._stream_id = request.stream_id
- return request
-
- def send_request(self, message):
- # TODO: implement flow control and WINDOW_UPDATE frames
- self.server_conn.send(self.server_protocol.assemble(message))
-
- def read_response(self, request_method):
- return HTTPResponse.from_protocol(
- self.server_protocol,
- request_method=request_method,
- body_size_limit=self.config.body_size_limit,
- include_body=True,
- stream_id=self._stream_id
- )
-
- def send_response(self, message):
- # TODO: implement flow control and WINDOW_UPDATE frames
- self.client_conn.send(self.client_protocol.assemble(message))
-
- def connect(self):
- self.ctx.connect()
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def reconnect(self):
- self.ctx.reconnect()
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def set_server(self, *args, **kwargs):
- self.ctx.set_server(*args, **kwargs)
- self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
- unhandled_frame_cb=self.handle_unexpected_frame)
- self.server_protocol.perform_connection_preface()
-
- def __call__(self):
- self.server_protocol.perform_connection_preface()
- layer = HttpLayer(self, self.mode)
- layer()
-
- def handle_unexpected_frame(self, frm):
- self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info")
-
-
-def make_error_response(status_code, message, headers=None):
- response = status_codes.RESPONSES.get(status_code, "Unknown")
- body = """
- <html>
- <head>
- <title>%d %s</title>
- </head>
- <body>%s</body>
- </html>
- """.strip() % (status_code, response, message)
-
- if not headers:
- headers = odict.ODictCaseless()
- headers["Server"] = [version.NAMEVERSION]
- headers["Connection"] = ["close"]
- headers["Content-Length"] = [len(body)]
- headers["Content-Type"] = ["text/html"]
-
- return HTTPResponse(
- (1, 1), # FIXME: Should be a string.
- status_code,
- response,
- headers,
- body,
- )
-
-
-def make_connect_request(address):
- address = Address.wrap(address)
- return HTTPRequest(
- "authority", "CONNECT", None, address.host, address.port, None, (1, 1),
- odict.ODictCaseless(), ""
- )
-
-
-def make_connect_response(httpversion):
- headers = odict.ODictCaseless([
- ["Content-Length", "0"],
- ["Proxy-Agent", version.NAMEVERSION]
- ])
- return HTTPResponse(
- httpversion,
- 200,
- "Connection established",
- headers,
- "",
- )
-
-
-class ConnectServerConnection(object):
- """
- "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
- """
-
- def __init__(self, address, ctx):
- self.address = tcp.Address.wrap(address)
- self._ctx = ctx
-
- @property
- def via(self):
- return self._ctx.server_conn
-
- def __getattr__(self, item):
- return getattr(self.via, item)
-
-
-class UpstreamConnectLayer(Layer):
- def __init__(self, ctx, connect_request):
- super(UpstreamConnectLayer, self).__init__(ctx)
- self.connect_request = connect_request
- self.server_conn = ConnectServerConnection(
- (connect_request.host, connect_request.port),
- self.ctx
- )
-
- def __call__(self):
- layer = self.ctx.next_layer(self)
- layer()
-
- def connect(self):
- if not self.server_conn:
- self.ctx.connect()
- self.send_request(self.connect_request)
- else:
- pass # swallow the message
-
- def reconnect(self):
- self.ctx.reconnect()
- self.send_request(self.connect_request)
- resp = self.read_response("CONNECT")
- if resp.code != 200:
- raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
-
- def set_server(self, address, server_tls=None, sni=None, depth=1):
- if depth == 1:
- if self.ctx.server_conn:
- self.ctx.reconnect()
- address = Address.wrap(address)
- self.connect_request.host = address.host
- self.connect_request.port = address.port
- self.server_conn.address = address
- else:
- self.ctx.set_server(address, server_tls, sni, depth - 1)
-
-
-class HttpLayer(Layer):
- def __init__(self, ctx, mode):
- super(HttpLayer, self).__init__(ctx)
- self.mode = mode
- self.__original_server_conn = None
- "Contains the original destination in transparent mode, which needs to be restored"
- "if an inline script modified the target server for a single http request"
-
- def __call__(self):
- if self.mode == "transparent":
- self.__original_server_conn = self.server_conn
- while True:
- try:
- flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
-
- try:
- request = self.read_request()
- except tcp.NetLibError:
- # don't throw an error for disconnects that happen
- # before/between requests.
- return
-
- self.log("request", "debug", [repr(request)])
-
- # Handle Proxy Authentication
- self.authenticate(request)
-
- # Regular Proxy Mode: Handle CONNECT
- if self.mode == "regular" and request.form_in == "authority":
- self.handle_regular_mode_connect(request)
- return
-
- # Make sure that the incoming request matches our expectations
- self.validate_request(request)
-
- flow.request = request
- self.process_request_hook(flow)
-
- if not flow.response:
- self.establish_server_connection(flow)
- self.get_response_from_server(flow)
-
- self.send_response_to_client(flow)
-
- if self.check_close_connection(flow):
- return
-
- # TODO: Implement HTTP Upgrade
-
- # Upstream Proxy Mode: Handle CONNECT
- if flow.request.form_in == "authority" and flow.response.code == 200:
- self.handle_upstream_mode_connect(flow.request.copy())
- return
-
- except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
- if flow.request and not flow.response:
- flow.error = Error(repr(e))
- self.channel.ask("error", flow)
- try:
- self.send_response(make_error_response(
- getattr(e, "code", 502),
- repr(e)
- ))
- except NetLibError:
- pass
- if isinstance(e, ProtocolException):
- raise e
- else:
- raise ProtocolException("Error in HTTP connection: %s" % repr(e), e)
- finally:
- flow.live = False
-
- def handle_regular_mode_connect(self, request):
- self.set_server((request.host, request.port))
- self.send_response(make_connect_response(request.httpversion))
- layer = self.ctx.next_layer(self)
- layer()
-
- def handle_upstream_mode_connect(self, connect_request):
- layer = UpstreamConnectLayer(self, connect_request)
- layer()
-
- def check_close_connection(self, flow):
- """
- Checks if the connection should be closed depending on the HTTP
- semantics. Returns True, if so.
- """
-
- # TODO: add logic for HTTP/2
-
- close_connection = (
- http1.HTTP1Protocol.connection_close(
- flow.request.httpversion,
- flow.request.headers
- ) or http1.HTTP1Protocol.connection_close(
- flow.response.httpversion,
- flow.response.headers
- ) or http1.HTTP1Protocol.expected_http_body_size(
- flow.response.headers,
- False,
- flow.request.method,
- flow.response.code) == -1
- )
- if flow.request.form_in == "authority" and flow.response.code == 200:
- # Workaround for
- # https://github.com/mitmproxy/mitmproxy/issues/313: Some
- # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
- # and no Content-Length header
-
- return False
- return close_connection
-
- def send_response_to_client(self, flow):
- if not (self.supports_streaming and flow.response.stream):
- # no streaming:
- # we already received the full response from the server and can
- # send it to the client straight away.
- self.send_response(flow.response)
- else:
- # streaming:
- # First send the headers and then transfer the response incrementally
- self.send_response_headers(flow.response)
- chunks = self.read_response_body(
- flow.response.headers,
- flow.request.method,
- flow.response.code,
- max_chunk_size=4096
- )
- if callable(flow.response.stream):
- chunks = flow.response.stream(chunks)
- self.send_response_body(flow.response, chunks)
- flow.response.timestamp_end = utils.timestamp()
-
- def get_response_from_server(self, flow):
- def get_response():
- self.send_request(flow.request)
- if self.supports_streaming:
- flow.response = self.read_response_headers()
- else:
- flow.response = self.read_response()
-
- try:
- get_response()
- except (tcp.NetLibError, HttpErrorConnClosed) as v:
- self.log(
- "server communication error: %s" % repr(v),
- level="debug"
- )
- # In any case, we try to reconnect at least once. This is
- # necessary because it might be possible that we already
- # initiated an upstream connection after clientconnect that
- # has already been expired, e.g consider the following event
- # log:
- # > clientconnect (transparent mode destination known)
- # > serverconnect (required for client tls handshake)
- # > read n% of large request
- # > server detects timeout, disconnects
- # > read (100-n)% of large request
- # > send large request upstream
- self.reconnect()
- get_response()
-
- # call the appropriate script hook - this is an opportunity for an
- # inline script to set flow.stream = True
- flow = self.channel.ask("responseheaders", flow)
- if flow is None or flow == KILL:
- raise Kill()
-
- if self.supports_streaming:
- if flow.response.stream:
- flow.response.content = CONTENT_MISSING
- else:
- flow.response.content = "".join(self.read_response_body(
- flow.response.headers,
- flow.request.method,
- flow.response.code
- ))
- flow.response.timestamp_end = utils.timestamp()
-
- # no further manipulation of self.server_conn beyond this point
- # we can safely set it as the final attribute value here.
- flow.server_conn = self.server_conn
-
- self.log(
- "response",
- "debug",
- [repr(flow.response)]
- )
- response_reply = self.channel.ask("response", flow)
- if response_reply is None or response_reply == KILL:
- raise Kill()
-
- def process_request_hook(self, flow):
- # Determine .scheme, .host and .port attributes for inline scripts.
- # For absolute-form requests, they are directly given in the request.
- # For authority-form requests, we only need to determine the request scheme.
- # For relative-form requests, we need to determine host and port as
- # well.
- if self.mode == "regular":
- pass # only absolute-form at this point, nothing to do here.
- elif self.mode == "upstream":
- if flow.request.form_in == "authority":
- flow.request.scheme = "http" # pseudo value
- else:
- flow.request.host = self.__original_server_conn.address.host
- flow.request.port = self.__original_server_conn.address.port
- flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http"
-
- request_reply = self.channel.ask("request", flow)
- if request_reply is None or request_reply == KILL:
- raise Kill()
- if isinstance(request_reply, HTTPResponse):
- flow.response = request_reply
- return
-
- def establish_server_connection(self, flow):
- address = tcp.Address((flow.request.host, flow.request.port))
- tls = (flow.request.scheme == "https")
-
- if self.mode == "regular" or self.mode == "transparent":
- # If there's an existing connection that doesn't match our expectations, kill it.
- if address != self.server_conn.address or tls != self.server_conn.ssl_established:
- self.set_server(address, tls, address.host)
- # Establish connection is neccessary.
- if not self.server_conn:
- self.connect()
-
- # SetServer is not guaranteed to work with TLS:
- # If there's not TlsLayer below which could catch the exception,
- # TLS will not be established.
- if tls and not self.server_conn.tls_established:
- raise ProtocolException(
- "Cannot upgrade to SSL, no TLS layer on the protocol stack.")
- else:
- if not self.server_conn:
- self.connect()
- if tls:
- raise HttpException("Cannot change scheme in upstream proxy mode.")
- """
- # This is a very ugly (untested) workaround to solve a very ugly problem.
- if self.server_conn and self.server_conn.tls_established and not ssl:
- self.reconnect()
- elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
- if self.server_conn.tls_established:
- self.reconnect()
-
- self.send_request(make_connect_request(address))
- tls_layer = TlsLayer(self, False, True)
- tls_layer._establish_tls_with_server()
- """
-
- def validate_request(self, request):
- if request.form_in == "absolute" and request.scheme != "http":
- self.send_response(
- make_error_response(400, "Invalid request scheme: %s" % request.scheme))
- raise HttpException("Invalid request scheme: %s" % request.scheme)
-
- expected_request_forms = {
- "regular": ("absolute",), # an authority request would already be handled.
- "upstream": ("authority", "absolute"),
- "transparent": ("relative",)
- }
-
- allowed_request_forms = expected_request_forms[self.mode]
- if request.form_in not in allowed_request_forms:
- err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
- " or ".join(allowed_request_forms), request.form_in
- )
- self.send_response(make_error_response(400, err_message))
- raise HttpException(err_message)
-
- if self.mode == "regular":
- request.form_out = "relative"
-
- def authenticate(self, request):
- if self.config.authenticator:
- if self.config.authenticator.authenticate(request.headers):
- self.config.authenticator.clean(request.headers)
- else:
- self.send_response(make_error_response(
- 407,
- "Proxy Authentication Required",
- odict.ODictCaseless(
- [
- [k, v] for k, v in
- self.config.authenticator.auth_challenge_headers().items()
- ])
- ))
- raise InvalidCredentials("Proxy Authentication Required")
diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py
index 709654cb..d5297cb1 100644
--- a/libmproxy/proxy/__init__.py
+++ b/libmproxy/proxy/__init__.py
@@ -1,11 +1,9 @@
from __future__ import (absolute_import, print_function, division)
-from .primitives import Log, Kill
+from .server import ProxyServer, DummyServer
from .config import ProxyConfig
-from .connection import ClientConnection, ServerConnection
__all__ = [
- "Log", "Kill",
+ "ProxyServer", "DummyServer",
"ProxyConfig",
- "ClientConnection", "ServerConnection"
-] \ No newline at end of file
+]
diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py
index b360abbd..65029087 100644
--- a/libmproxy/proxy/config.py
+++ b/libmproxy/proxy/config.py
@@ -6,9 +6,9 @@ from OpenSSL import SSL
from netlib import certutils, tcp
from netlib.http import authentication
+from netlib.tcp import Address, sslversion_choices
from .. import utils, platform
-from netlib.tcp import Address, sslversion_choices
CONF_BASENAME = "mitmproxy"
CA_DIR = "~/.mitmproxy"
diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/proxy/modes/__init__.py
index 61b9a77e..f014ed98 100644
--- a/libmproxy/protocol2/__init__.py
+++ b/libmproxy/proxy/modes/__init__.py
@@ -1,13 +1,12 @@
from __future__ import (absolute_import, print_function, division)
-from .root_context import RootContext
-from .socks_proxy import Socks5Proxy
-from .reverse_proxy import ReverseProxy
from .http_proxy import HttpProxy, HttpUpstreamProxy
+from .reverse_proxy import ReverseProxy
+from .socks_proxy import Socks5Proxy
from .transparent_proxy import TransparentProxy
-from .http import make_error_response
__all__ = [
- "RootContext",
- "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy",
- "make_error_response"
+ "HttpProxy", "HttpUpstreamProxy",
+ "ReverseProxy",
+ "Socks5Proxy",
+ "TransparentProxy"
]
diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py
index 2876c022..90c54cc6 100644
--- a/libmproxy/protocol2/http_proxy.py
+++ b/libmproxy/proxy/modes/http_proxy.py
@@ -1,6 +1,6 @@
from __future__ import (absolute_import, print_function, division)
-from .layer import Layer, ServerConnectionMixin
+from ...protocol import Layer, ServerConnectionMixin
class HttpProxy(Layer, ServerConnectionMixin):
diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py
index 3ca998d5..b57ac5eb 100644
--- a/libmproxy/protocol2/reverse_proxy.py
+++ b/libmproxy/proxy/modes/reverse_proxy.py
@@ -1,6 +1,6 @@
from __future__ import (absolute_import, print_function, division)
-from .layer import Layer, ServerConnectionMixin
+from ...protocol import Layer, ServerConnectionMixin
class ReverseProxy(Layer, ServerConnectionMixin):
diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py
index 525520e8..ebaf939e 100644
--- a/libmproxy/protocol2/socks_proxy.py
+++ b/libmproxy/proxy/modes/socks_proxy.py
@@ -2,8 +2,9 @@ from __future__ import (absolute_import, print_function, division)
from netlib import socks
from netlib.tcp import NetLibError
-from ..exceptions import Socks5Exception
-from .layer import Layer, ServerConnectionMixin
+
+from ...exceptions import Socks5Exception
+from ...protocol import Layer, ServerConnectionMixin
class Socks5Proxy(Layer, ServerConnectionMixin):
diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py
index e6ebf115..96ad86c4 100644
--- a/libmproxy/protocol2/transparent_proxy.py
+++ b/libmproxy/proxy/modes/transparent_proxy.py
@@ -1,8 +1,8 @@
from __future__ import (absolute_import, print_function, division)
-from ..exceptions import ProtocolException
-from .. import platform
-from .layer import Layer, ServerConnectionMixin
+from ... import platform
+from ...exceptions import ProtocolException
+from ...protocol import Layer, ServerConnectionMixin
class TransparentProxy(Layer, ServerConnectionMixin):
diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py
deleted file mode 100644
index 2e440fe8..00000000
--- a/libmproxy/proxy/primitives.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from __future__ import absolute_import
-import collections
-from netlib import socks, tcp
-
-
-class Log(object):
- def __init__(self, msg, level="info"):
- self.msg = msg
- self.level = level
-
-
-class Kill(Exception):
- """
- Kill a connection.
- """ \ No newline at end of file
diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/proxy/root_context.py
index daea54bd..35909612 100644
--- a/libmproxy/protocol2/root_context.py
+++ b/libmproxy/proxy/root_context.py
@@ -3,12 +3,10 @@ from __future__ import (absolute_import, print_function, division)
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
-from .rawtcp import RawTcpLayer
-from .tls import TlsLayer, is_tls_record_magic
-from .http import Http1Layer, Http2Layer
-from .layer import ServerConnectionMixin
-from .http_proxy import HttpProxy, HttpUpstreamProxy
-from .reverse_proxy import ReverseProxy
+from ..protocol import (
+ RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin
+)
+from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy
class RootContext(object):
@@ -35,7 +33,7 @@ class RootContext(object):
# 1. Check for --ignore.
if self.config.check_ignore(top_layer.server_conn.address):
- return RawTcpLayer(top_layer, logging=False)
+ return RawTCPLayer(top_layer, logging=False)
d = top_layer.client_conn.rfile.peek(3)
client_tls = is_tls_record_magic(d)
@@ -61,7 +59,7 @@ class RootContext(object):
# 4. Check for --tcp
if self.config.check_tcp(top_layer.server_conn.address):
- return RawTcpLayer(top_layer)
+ return RawTCPLayer(top_layer)
# 5. Check for TLS ALPN (HTTP1/HTTP2)
if isinstance(top_layer, TlsLayer):
diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py
index 5abd0877..2a451ba1 100644
--- a/libmproxy/proxy/server.py
+++ b/libmproxy/proxy/server.py
@@ -3,14 +3,15 @@ from __future__ import absolute_import, print_function
import traceback
import sys
import socket
+
from netlib import tcp
from netlib.http.http1 import HTTP1Protocol
from netlib.tcp import NetLibError
-
-from .. import protocol2
from ..exceptions import ProtocolException, ServerException
-from .primitives import Log, Kill
-from .connection import ClientConnection
+from ..protocol import Log, Kill
+from ..models import ClientConnection, make_error_response
+from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy
+from .root_context import RootContext
class DummyServer:
@@ -71,7 +72,7 @@ class ConnectionHandler(object):
"""@type: libmproxy.controller.Channel"""
def _create_root_layer(self):
- root_context = protocol2.RootContext(
+ root_context = RootContext(
self.client_conn,
self.config,
self.channel
@@ -79,23 +80,23 @@ class ConnectionHandler(object):
mode = self.config.mode
if mode == "upstream":
- return protocol2.HttpUpstreamProxy(
+ return HttpUpstreamProxy(
root_context,
self.config.upstream_server.address
)
elif mode == "transparent":
- return protocol2.TransparentProxy(root_context)
+ return TransparentProxy(root_context)
elif mode == "reverse":
server_tls = self.config.upstream_server.scheme == "https"
- return protocol2.ReverseProxy(
+ return ReverseProxy(
root_context,
self.config.upstream_server.address,
server_tls
)
elif mode == "socks5":
- return protocol2.Socks5Proxy(root_context)
+ return Socks5Proxy(root_context)
elif mode == "regular":
- return protocol2.HttpProxy(root_context)
+ return HttpProxy(root_context)
elif callable(mode): # pragma: nocover
return mode(root_context)
else: # pragma: nocover
@@ -116,7 +117,7 @@ class ConnectionHandler(object):
# we send an HTTP error response, which is both
# understandable by HTTP clients and humans.
try:
- error_response = protocol2.make_error_response(502, repr(e))
+ error_response = make_error_response(502, repr(e))
self.client_conn.send(HTTP1Protocol().assemble(error_response))
except NetLibError:
pass
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-06 01:15:34 +0000