1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
|
import h2.exceptions
import netlib.exceptions
import time
import traceback
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy.protocol import base
from mitmproxy.protocol import websockets as pwebsockets
from netlib import http
from netlib import tcp
from netlib import websockets
class _HttpTransmissionLayer(base.Layer):
def read_request_headers(self):
raise NotImplementedError()
def read_request_body(self, request):
raise NotImplementedError()
def read_request(self):
request = self.read_request_headers()
request.data.content = b"".join(
self.read_request_body(request)
)
request.timestamp_end = time.time()
return request
def send_request(self, request):
raise NotImplementedError()
def read_response_headers(self):
raise NotImplementedError()
def read_response_body(self, request, response):
raise NotImplementedError()
yield "this is a generator" # pragma: no cover
def read_response(self, request):
response = self.read_response_headers()
response.data.content = b"".join(
self.read_response_body(request, response)
)
return response
def send_response(self, response):
if response.data.content is None:
raise netlib.exceptions.HttpException("Cannot assemble flow with missing content")
self.send_response_headers(response)
self.send_response_body(response, [response.data.content])
def send_response_headers(self, response):
raise NotImplementedError()
def send_response_body(self, response, chunks):
raise NotImplementedError()
def check_close_connection(self, flow):
raise NotImplementedError()
class ConnectServerConnection:
"""
"Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
"""
def __init__(self, address, ctx):
self.address = tcp.Address.wrap(address)
self._ctx = ctx
@property
def via(self):
return self._ctx.server_conn
def __getattr__(self, item):
return getattr(self.via, item)
def __bool__(self):
return bool(self.via)
class UpstreamConnectLayer(base.Layer):
def __init__(self, ctx, connect_request):
super().__init__(ctx)
self.connect_request = connect_request
self.server_conn = ConnectServerConnection(
(connect_request.host, connect_request.port),
self.ctx
)
def __call__(self):
layer = self.ctx.next_layer(self)
layer()
def _send_connect_request(self):
self.send_request(self.connect_request)
resp = self.read_response(self.connect_request)
if resp.status_code != 200:
raise exceptions.ProtocolException("Reconnect: Upstream server refuses CONNECT request")
def connect(self):
if not self.server_conn:
self.ctx.connect()
self._send_connect_request()
else:
pass # swallow the message
def change_upstream_proxy_server(self, address):
if address != self.server_conn.via.address:
self.ctx.set_server(address)
def set_server(self, address):
if self.ctx.server_conn:
self.ctx.disconnect()
address = tcp.Address.wrap(address)
self.connect_request.host = address.host
self.connect_request.port = address.port
self.server_conn.address = address
class HttpLayer(base.Layer):
def __init__(self, ctx, mode):
super().__init__(ctx)
self.mode = mode
self.__initial_server_conn = None
"Contains the original destination in transparent mode, which needs to be restored"
"if an inline script modified the target server for a single http request"
# We cannot rely on server_conn.tls_established,
# see https://github.com/mitmproxy/mitmproxy/issues/925
self.__initial_server_tls = None
# Requests happening after CONNECT do not need Proxy-Authorization headers.
self.http_authenticated = False
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
try:
request = self.get_request_from_client(flow)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except netlib.exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except netlib.exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return
flow.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
return
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
flow.request.headers["Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(flow)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", flow)
if not flow.response:
self.establish_server_connection(
flow.request.host,
flow.request.port,
flow.request.scheme
)
self.get_response_from_server(flow)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", flow)
self.log("response", "debug", [repr(flow.response)])
self.channel.ask("response", flow)
self.send_response_to_client(flow)
if self.check_close_connection(flow):
return
# Handle 101 Switching Protocols
if flow.response.status_code == 101:
return self.handle_101_switching_protocols(flow)
# Upstream Proxy Mode: Handle CONNECT
if flow.request.first_line_format == "authority" and flow.response.status_code == 200:
self.handle_upstream_mode_connect(flow.request.copy())
return
except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not flow.response:
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
return
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if flow:
flow.live = False
def get_request_from_client(self, flow):
request = self.read_request()
flow.request = request
self.channel.ask("requestheaders", flow)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2 here.
self.send_response(models.expect_continue_response)
request.headers.pop("expect")
request.body = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
return request
def send_error_response(self, code, message, headers=None):
try:
response = models.make_error_response(code, message, headers)
self.send_response(response)
except (netlib.exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
self.log(traceback.format_exc(), "debug")
def change_upstream_proxy_server(self, address):
# Make set_upstream_proxy_server always available,
# even if there's no UpstreamConnectLayer
if address != self.server_conn.address:
return self.set_server(address)
def handle_regular_mode_connect(self, request):
self.http_authenticated = True
self.set_server((request.host, request.port))
self.send_response(models.make_connect_response(request.data.http_version))
layer = self.ctx.next_layer(self)
layer()
def handle_upstream_mode_connect(self, connect_request):
layer = UpstreamConnectLayer(self, connect_request)
layer()
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(flow.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(flow.response)
chunks = self.read_response_body(
flow.request,
flow.response
)
if callable(flow.response.stream):
chunks = flow.response.stream(chunks)
self.send_response_body(flow.response, chunks)
flow.response.timestamp_end = time.time()
def get_response_from_server(self, flow):
def get_response():
self.send_request(flow.request)
flow.response = self.read_response_headers()
try:
get_response()
except netlib.exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for an
# inline script to set flow.stream = True
self.channel.ask("responseheaders", flow)
if flow.response.stream:
flow.response.data.content = None
else:
flow.response.data.content = b"".join(self.read_response_body(
flow.request,
flow.response
))
flow.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.server_conn
def process_request_hook(self, flow):
# Determine .scheme, .host and .port attributes for inline scripts.
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as
# well.
if self.mode == "regular":
pass # only absolute-form at this point, nothing to do here.
elif self.mode == "upstream":
pass
else:
# Setting request.host also updates the host header, which we want to preserve
host_header = flow.request.headers.get("host", None)
flow.request.host = self.__initial_server_conn.address.host
flow.request.port = self.__initial_server_conn.address.port
if host_header:
flow.request.headers["host"] = host_header
flow.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", flow)
def establish_server_connection(self, host, port, scheme):
address = tcp.Address((host, port))
tls = (scheme == "https")
if self.mode == "regular" or self.mode == "transparent":
# If there's an existing connection that doesn't match our expectations, kill it.
if address != self.server_conn.address or tls != self.server_tls:
self.set_server(address)
self.set_server_tls(tls, address.host)
# Establish connection is neccessary.
if not self.server_conn:
self.connect()
else:
if not self.server_conn:
self.connect()
if tls:
raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
if self.server_conn and self.server_conn.tls_established and not ssl:
self.disconnect()
self.connect()
elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
if self.server_conn.tls_established:
self.disconnect()
self.connect()
self.send_request(make_connect_request(address))
tls_layer = TlsLayer(self, False, True)
tls_layer._establish_tls_with_server()
"""
def validate_request(self, request):
if request.first_line_format == "absolute" and request.scheme != "http":
raise netlib.exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
"regular": ("authority", "absolute",),
"upstream": ("authority", "absolute"),
"transparent": ("relative",)
}
allowed_request_forms = expected_request_forms[self.mode]
if request.first_line_format not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.first_line_format
)
raise netlib.exceptions.HttpException(err_message)
if self.mode == "regular" and request.first_line_format == "absolute":
request.first_line_format = "relative"
def authenticate(self, request):
if self.config.authenticator:
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
if self.mode == "transparent":
self.send_response(models.make_error_response(
401,
"Authentication Required",
http.Headers(**self.config.authenticator.auth_challenge_headers())
))
else:
self.send_response(models.make_error_response(
407,
"Proxy Authentication Required",
http.Headers(**self.config.authenticator.auth_challenge_headers())
))
return False
return True
def handle_101_switching_protocols(self, flow):
"""
Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
"""
# Check for WebSockets handshake
is_websockets = (
flow and
websockets.check_handshake(flow.request.headers) and
websockets.check_handshake(flow.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
"info"
)
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, flow)
else:
layer = self.ctx.next_layer(self)
layer()
|