Add requestheaders event

HTTP1 only for now, HTTP2 coming next.

5 files changed, 32 insertions, 17 deletions

diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py
index ea084973..7996eb46 100644
--- a/mitmproxy/controller.py
+++ b/mitmproxy/controller.py
@@ -25,6 +25,7 @@ Events = frozenset([
     "tcp_close",
 
     "request",
+    "requestheaders",
     "response",
     "responseheaders",
 
diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py
index de9d47f1..7149fec9 100644
--- a/mitmproxy/flow/master.py
+++ b/mitmproxy/flow/master.py
@@ -18,6 +18,7 @@ from mitmproxy.protocol import http_replay
 def event_sequence(f):
     if isinstance(f, models.HTTPFlow):
         if f.request:
+            yield "requestheaders", f
             yield "request", f
         if f.response:
             yield "responseheaders", f
@@ -216,6 +217,10 @@ class FlowMaster(controller.Master):
         self.state.update_flow(f)
 
     @controller.handler
+    def requestheaders(self, f):
+        pass
+
+    @controller.handler
     def request(self, f):
         if f.live:
             app = self.apps.get(f.request)
diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py
index 4a30edb7..744733a8 100644
--- a/mitmproxy/protocol/http.py
+++ b/mitmproxy/protocol/http.py
@@ -16,23 +16,23 @@ from netlib import websockets
 
 
 class _HttpTransmissionLayer(base.Layer):
-
-    def read_request(self):
+    def read_request_headers(self):
         raise NotImplementedError()
 
     def read_request_body(self, request):
         raise NotImplementedError()
 
+    def read_request(self):
+        request = self.read_request_headers()
+        request.data.content = b"".join(
+            self.read_request_body(request)
+        )
+        request.timestamp_end = time.time()
+        return request
+
     def send_request(self, request):
         raise NotImplementedError()
 
-    def read_response(self, request):
-        response = self.read_response_headers()
-        response.data.content = b"".join(
-            self.read_response_body(request, response)
-        )
-        return response
-
     def read_response_headers(self):
         raise NotImplementedError()
 
@@ -40,6 +40,13 @@ class _HttpTransmissionLayer(base.Layer):
         raise NotImplementedError()
         yield "this is a generator"  # pragma: no cover
 
+    def read_response(self, request):
+        response = self.read_response_headers()
+        response.data.content = b"".join(
+            self.read_response_body(request, response)
+        )
+        return response
+
     def send_response(self, response):
         if response.data.content is None:
             raise netlib.exceptions.HttpException("Cannot assemble flow with missing content")
@@ -140,8 +147,9 @@ class HttpLayer(base.Layer):
             self.__initial_server_tls = self.server_tls
             self.__initial_server_conn = self.server_conn
         while True:
+            flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
             try:
-                request = self.get_request_from_client()
+                request = self.get_request_from_client(flow)
                 # Make sure that the incoming request matches our expectations
                 self.validate_request(request)
             except netlib.exceptions.HttpReadDisconnect:
@@ -168,7 +176,6 @@ class HttpLayer(base.Layer):
             if not (self.http_authenticated or self.authenticate(request)):
                 return
 
-            flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
             flow.request = request
 
             try:
@@ -243,13 +250,16 @@ class HttpLayer(base.Layer):
                 if flow:
                     flow.live = False
 
-    def get_request_from_client(self):
+    def get_request_from_client(self, flow):
         request = self.read_request()
+        flow.request = request
+        flow = self.channel.ask("requestheaders", flow)
         if request.headers.get("expect", "").lower() == "100-continue":
             # TODO: We may have to use send_response_headers for HTTP2 here.
             self.send_response(models.expect_continue_response)
             request.headers.pop("expect")
             request.body = b"".join(self.read_request_body(request))
+            request.timestamp_end = time.time()
         return request
 
     def send_error_response(self, code, message, headers=None):
diff --git a/mitmproxy/protocol/http1.py b/mitmproxy/protocol/http1.py
index 8698fe31..2f8a15de 100644
--- a/mitmproxy/protocol/http1.py
+++ b/mitmproxy/protocol/http1.py
@@ -11,11 +11,10 @@ class Http1Layer(http._HttpTransmissionLayer):
         super(Http1Layer, self).__init__(ctx)
         self.mode = mode
 
-    def read_request(self):
-        req = http1.read_request(
-            self.client_conn.rfile, body_size_limit=self.config.options.body_size_limit
+    def read_request_headers(self):
+        return models.HTTPRequest.wrap(
+            http1.read_request_head(self.client_conn.rfile)
         )
-        return models.HTTPRequest.wrap(req)
 
     def read_request_body(self, request):
         expected_size = http1.expected_http_body_size(request)
diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py
index f92765ee..261adb65 100644
--- a/test/mitmproxy/builtins/test_script.py
+++ b/test/mitmproxy/builtins/test_script.py
@@ -175,7 +175,7 @@ class TestScriptLoader(mastertest.MasterTest):
                 ), [f]
             )
         evts = [i[1] for i in sc.ns.call_log]
-        assert evts == ['start', 'request', 'responseheaders', 'response', 'done']
+        assert evts == ['start', 'requestheaders', 'request', 'responseheaders', 'response', 'done']
 
         with m.handlecontext():
             tutils.raises(