diff options
| -rw-r--r-- | netlib/http2/frame.py | 24 | ||||
| -rw-r--r-- | netlib/http2/protocol.py | 99 | ||||
| -rw-r--r-- | netlib/tcp.py | 18 | ||||
| -rw-r--r-- | test/http2/test_http2_protocol.py | 17 | ||||
| -rw-r--r-- | test/test_tcp.py | 41 |
5 files changed, 126 insertions, 73 deletions
diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 4a305d82..98ced904 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -113,16 +113,13 @@ class Frame(object): def payload_human_readable(self): # pragma: no cover raise NotImplementedError() - def human_readable(self): + def human_readable(self, direction="-"): + self.length = len(self.payload_bytes()) + return "\n".join([ - "============================================================", - "length: %d bytes" % self.length, - "type: %s (%#x)" % (self.__class__.__name__, self.TYPE), - "flags: %#x" % self.flags, - "stream_id: %#x" % self.stream_id, - "------------------------------------------------------------", + "%s: %s | length: %d | flags: %#x | stream_id: %d" % (direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), - "============================================================", + "===============================================================", ]) def __eq__(self, other): @@ -461,7 +458,10 @@ class PushPromiseFrame(Frame): s.append("padding: %d" % self.pad_length) s.append("promised stream: %#x" % self.promised_stream) - s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) @@ -605,7 +605,11 @@ class ContinuationFrame(Frame): return self.header_block_fragment def payload_human_readable(self): - return "header_block_fragment: %s" % str(self.header_block_fragment) + s = [] + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) _FRAME_CLASSES = [ DataFrame, diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 1e722dfb..ac89bac4 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -31,7 +31,7 @@ class HTTP2Protocol(object): ALPN_PROTO_H2 = 'h2' - def __init__(self, tcp_handler, is_server=False): + def __init__(self, tcp_handler, is_server=False, dump_frames=False): self.tcp_handler = tcp_handler self.is_server = is_server @@ -40,6 +40,7 @@ class HTTP2Protocol(object): self.encoder = Encoder() self.decoder = Decoder() self.connection_preface_performed = False + self.dump_frames = dump_frames def check_alpn(self): alp = self.tcp_handler.get_alpn_proto_negotiated() @@ -48,16 +49,19 @@ class HTTP2Protocol(object): "HTTP2Protocol can not handle unknown ALP: %s" % alp) return True - def _receive_settings(self): - frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - assert isinstance(frm, frame.SettingsFrame) - self._apply_settings(frm.settings) + def _receive_settings(self, hide=False): + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + break - def _read_settings_ack(self): - settings_ack_frame = self.read_frame() - assert isinstance(settings_ack_frame, frame.SettingsFrame) - assert settings_ack_frame.flags & frame.Frame.FLAG_ACK - assert len(settings_ack_frame.settings) == 0 + def _read_settings_ack(self, hide=False): # pragma no cover + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + assert settings_ack_frame.flags & frame.Frame.FLAG_ACK + assert len(settings_ack_frame.settings) == 0 + break def perform_server_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -67,9 +71,8 @@ class HTTP2Protocol(object): magic = self.tcp_handler.rfile.safe_read(magic_length) assert magic == self.CLIENT_CONNECTION_PREFACE - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def perform_client_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -77,9 +80,8 @@ class HTTP2Protocol(object): self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def next_stream_id(self): if self.current_stream_id is None: @@ -93,30 +95,37 @@ class HTTP2Protocol(object): self.current_stream_id += 2 return self.current_stream_id - def send_frame(self, frame): - raw_bytes = frame.to_bytes() + def send_frame(self, frm, hide=False): + raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() + if not hide and self.dump_frames: # pragma no cover + print(frm.human_readable(">>")) - def read_frame(self): + def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if isinstance(frm, frame.SettingsFrame): - self._apply_settings(frm.settings) + if not hide and self.dump_frames: # pragma no cover + print(frm.human_readable("<<")) + if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: + self._apply_settings(frm.settings, hide) return frm - def _apply_settings(self, settings): + def _apply_settings(self, settings, hide=False): for setting, value in settings.items(): old_value = self.http2_settings[setting] if not old_value: old_value = '-' - self.http2_settings[setting] = value self.send_frame( frame.SettingsFrame( state=self, - flags=frame.Frame.FLAG_ACK)) + flags=frame.Frame.FLAG_ACK), + hide) + + # be liberal in what we expect from the other end + # to be more strict use: self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -127,12 +136,16 @@ class HTTP2Protocol(object): header_block_fragment = self.encoder.encode(headers) - bytes = frame.HeadersFrame( + frm = frame.HeadersFrame( state=self, flags=flags, stream_id=stream_id, - header_block_fragment=header_block_fragment).to_bytes() - return [bytes] + header_block_fragment=header_block_fragment) + + if self.dump_frames: # pragma no cover + print(frm.human_readable(">>")) + + return [frm.to_bytes()] def _create_body(self, body, stream_id): if body is None or len(body) == 0: @@ -141,18 +154,26 @@ class HTTP2Protocol(object): # TODO: implement max frame size checks and sending in chunks # TODO: implement flow-control window - bytes = frame.DataFrame( + frm = frame.DataFrame( state=self, flags=frame.Frame.FLAG_END_STREAM, stream_id=stream_id, - payload=body).to_bytes() - return [bytes] + payload=body) + + if self.dump_frames: # pragma no cover + print(frm.human_readable(">>")) + + return [frm.to_bytes()] + def create_request(self, method, path, headers=None, body=None): if headers is None: headers = [] authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host + if self.tcp_handler.address.port != 443: + authority += ":%d" % self.tcp_handler.address.port + headers = [ (b':method', bytes(method)), (b':path', bytes(path)), @@ -167,7 +188,7 @@ class HTTP2Protocol(object): self._create_body(body, stream_id))) def read_response(self): - headers, body = self._receive_transmission() + stream_id, headers, body = self._receive_transmission() return headers[':status'], headers, body def read_request(self): @@ -176,6 +197,7 @@ class HTTP2Protocol(object): def _receive_transmission(self): body_expected = True + stream_id = 0 header_block_fragment = b'' body = b'' @@ -183,10 +205,11 @@ class HTTP2Protocol(object): frm = self.read_frame() if isinstance(frm, frame.HeadersFrame)\ or isinstance(frm, frame.ContinuationFrame): + stream_id = frm.stream_id header_block_fragment += frm.header_block_fragment + if frm.flags & frame.Frame.FLAG_END_STREAM: + body_expected = False if frm.flags & frame.Frame.FLAG_END_HEADERS: - if frm.flags & frame.Frame.FLAG_END_STREAM: - body_expected = False break while body_expected: @@ -201,16 +224,18 @@ class HTTP2Protocol(object): for header, value in self.decoder.decode(header_block_fragment): headers[header] = value - return headers, body + return stream_id, headers, body - def create_response(self, code, headers=None, body=None): + def create_response(self, code, stream_id=None, headers=None, body=None): if headers is None: headers = [] headers = [(b':status', bytes(str(code)))] + headers - stream_id = self.next_stream_id() + if not stream_id: + stream_id = self.next_stream_id() return list(itertools.chain( self._create_headers(headers, stream_id, end_stream=(body is None)), - self._create_body(body, stream_id))) + self._create_body(body, stream_id), + )) diff --git a/netlib/tcp.py b/netlib/tcp.py index 953cef6e..cafc3ed9 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -22,11 +22,6 @@ TLSv1_METHOD = SSL.TLSv1_METHOD TLSv1_1_METHOD = SSL.TLSv1_1_METHOD TLSv1_2_METHOD = SSL.TLSv1_2_METHOD -OP_NO_SSLv2 = SSL.OP_NO_SSLv2 -OP_NO_SSLv3 = SSL.OP_NO_SSLv3 -VERIFY_NONE = SSL.VERIFY_NONE - - class NetLibError(Exception): pass @@ -374,8 +369,8 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, - options=(OP_NO_SSLv2 | OP_NO_SSLv3), - verify_options=VERIFY_NONE, + options=(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_COMPRESSION), + verify_options=SSL.VERIFY_NONE, ca_path=None, ca_pemfile=None, cipher_list=None, @@ -397,7 +392,7 @@ class _Connection(object): context.set_options(options) # Verify Options (NONE/PEER/PEER|FAIL_IF_... and trusted CAs) - if verify_options is not None and verify_options is not VERIFY_NONE: + if verify_options is not None and verify_options is not SSL.VERIFY_NONE: def verify_cert(conn, cert, errno, err_depth, is_cert_verified): if is_cert_verified: return True @@ -419,6 +414,9 @@ class _Connection(object): if cipher_list: try: context.set_cipher_list(cipher_list) + + # TODO: maybe change this to with newer pyOpenSSL APIs + context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) except SSL.Error as v: raise NetLibError("SSL cipher specification error: %s" % str(v)) @@ -529,7 +527,7 @@ class TCPClient(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class BaseHandler(_Connection): @@ -639,7 +637,7 @@ class BaseHandler(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class TCPServer(object): diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index 34c69fa9..9b49acd3 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -222,14 +222,14 @@ class TestCreateRequest(): def test_create_request_simple(self): bytes = http2.HTTP2Protocol(self.c).create_request('GET', '/') assert len(bytes) == 1 - assert bytes[0] == '00000c0105000000018284874187089d5c0b8170ff'.decode('hex') + assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex') def test_create_request_with_body(self): bytes = http2.HTTP2Protocol(self.c).create_request( 'GET', '/', [(b'foo', b'bar')], 'foobar') assert len(bytes) == 2 assert bytes[0] ==\ - '0000140104000000018284874187089d5c0b8170ff408294e7838c767f'.decode('hex') + '0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex') assert bytes[1] ==\ '000006000100000001666f6f626172'.decode('hex') @@ -300,8 +300,9 @@ class TestReadRequest(test.ServerTestBase): c.convert_to_ssl() protocol = http2.HTTP2Protocol(c, is_server=True) - headers, body = protocol.read_request() + stream_id, headers, body = protocol.read_request() + assert stream_id assert headers == {':method': 'GET', ':path': '/', ':scheme': 'https'} assert body == b'foobar' @@ -309,17 +310,17 @@ class TestReadRequest(test.ServerTestBase): class TestCreateResponse(): c = tcp.TCPClient(("127.0.0.1", 0)) - def test_create_request_simple(self): + def test_create_response_simple(self): bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response(200) assert len(bytes) == 1 assert bytes[0] ==\ '00000101050000000288'.decode('hex') - def test_create_request_with_body(self): + def test_create_response_with_body(self): bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response( - 200, [(b'foo', b'bar')], 'foobar') + 200, 1, [(b'foo', b'bar')], 'foobar') assert len(bytes) == 2 assert bytes[0] ==\ - '00000901040000000288408294e7838c767f'.decode('hex') + '00000901040000000188408294e7838c767f'.decode('hex') assert bytes[1] ==\ - '000006000100000002666f6f626172'.decode('hex') + '000006000100000001666f6f626172'.decode('hex') diff --git a/test/test_tcp.py b/test/test_tcp.py index 0cecaaa2..122c1f0f 100644 --- a/test/test_tcp.py +++ b/test/test_tcp.py @@ -41,6 +41,18 @@ class HangHandler(tcp.BaseHandler): time.sleep(1) +class ALPNHandler(tcp.BaseHandler): + sni = None + + def handle(self): + alp = self.get_alpn_proto_negotiated() + if alp: + self.wfile.write("%s" % alp) + else: + self.wfile.write("NONE") + self.wfile.flush() + + class TestServer(test.ServerTestBase): handler = EchoHandler @@ -416,30 +428,43 @@ class TestTimeOut(test.ServerTestBase): tutils.raises(tcp.NetLibTimeout, c.rfile.read, 10) -class TestALPN(test.ServerTestBase): - handler = EchoHandler +class TestALPNClient(test.ServerTestBase): + handler = ALPNHandler ssl = dict( - alpn_select="foobar" + alpn_select="bar" ) if OpenSSL._util.lib.Cryptography_HAS_ALPN: def test_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - c.convert_to_ssl(alpn_protos=["foobar"]) - assert c.get_alpn_proto_negotiated() == "foobar" + c.convert_to_ssl(alpn_protos=["foo", "bar", "fasel"]) + assert c.get_alpn_proto_negotiated() == "bar" + assert c.rfile.readline().strip() == "bar" def test_no_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - assert c.get_alpn_proto_negotiated() == None + c.convert_to_ssl() + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline().strip() == "NONE" else: def test_none_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - c.convert_to_ssl(alpn_protos=["foobar"]) - assert c.get_alpn_proto_negotiated() == None + c.convert_to_ssl(alpn_protos=["foo", "bar", "fasel"]) + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline() == "NONE" + +class TestNoSSLNoALPNClient(test.ServerTestBase): + handler = ALPNHandler + + def test_no_ssl_no_alpn(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline().strip() == "NONE" class TestSSLTimeOut(test.ServerTestBase): |