diff options
Diffstat (limited to 'netlib')
| -rw-r--r-- | netlib/h2/__init__.py | 1 | ||||
| -rw-r--r-- | netlib/h2/h2.py | 89 | ||||
| -rw-r--r-- | netlib/http2/__init__.py | 182 | ||||
| -rw-r--r-- | netlib/http2/frame.py (renamed from netlib/h2/frame.py) | 167 | ||||
| -rw-r--r-- | netlib/test.py | 30 |
5 files changed, 308 insertions, 161 deletions
diff --git a/netlib/h2/__init__.py b/netlib/h2/__init__.py deleted file mode 100644 index 9b4faa33..00000000 --- a/netlib/h2/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from __future__ import (absolute_import, print_function, division) diff --git a/netlib/h2/h2.py b/netlib/h2/h2.py deleted file mode 100644 index 707b1465..00000000 --- a/netlib/h2/h2.py +++ /dev/null @@ -1,89 +0,0 @@ -from .. import utils, odict, tcp -from frame import * - -# "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" -CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a' - -ERROR_CODES = utils.BiDi( - NO_ERROR=0x0, - PROTOCOL_ERROR=0x1, - INTERNAL_ERROR=0x2, - FLOW_CONTROL_ERROR=0x3, - SETTINGS_TIMEOUT=0x4, - STREAM_CLOSED=0x5, - FRAME_SIZE_ERROR=0x6, - REFUSED_STREAM=0x7, - CANCEL=0x8, - COMPRESSION_ERROR=0x9, - CONNECT_ERROR=0xa, - ENHANCE_YOUR_CALM=0xb, - INADEQUATE_SECURITY=0xc, - HTTP_1_1_REQUIRED=0xd -) - - -class H2Client(tcp.TCPClient): - ALPN_PROTO_H2 = b'h2' - - DEFAULT_SETTINGS = { - SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096, - SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1, - SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None, - SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ^ 16 - 1, - SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ^ 14, - SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None, - } - - def __init__(self, address, source_address=None): - super(H2Client, self).__init__(address, source_address) - self.settings = self.DEFAULT_SETTINGS.copy() - - def connect(self, send_preface=True): - super(H2Client, self).connect() - self.convert_to_ssl(alpn_protos=[self.ALPN_PROTO_H2]) - - alp = self.get_alpn_proto_negotiated() - if alp != b'h2': - raise NotImplementedError( - "H2Client can not handle unknown protocol: %s" % - alp) - print "-> Successfully negotiated 'h2' application layer protocol." - - if send_preface: - self.wfile.write(bytes(CLIENT_CONNECTION_PREFACE.decode('hex'))) - self.send_frame(SettingsFrame()) - - frame = Frame.from_file(self.rfile) - print frame.human_readable() - assert isinstance(frame, SettingsFrame) - self.apply_settings(frame.settings) - - print "-> Connection Preface completed." - - print "-> H2Client is ready..." - - def send_frame(self, frame): - self.wfile.write(frame.to_bytes()) - self.wfile.flush() - - def read_frame(self): - frame = Frame.from_file(self.rfile) - if isinstance(frame, SettingsFrame): - self.apply_settings(frame.settings) - - return frame - - def apply_settings(self, settings): - for setting, value in settings.items(): - old_value = self.settings[setting] - if not old_value: - old_value = '-' - - self.settings[setting] = value - print "-> Setting changed: %s to %d (was %s)" % - (SettingsFrame.SETTINGS.get_name(setting), - value, - str(old_value)) - - self.send_frame(SettingsFrame(flags=Frame.FLAG_ACK)) - print "-> New settings acknowledged." diff --git a/netlib/http2/__init__.py b/netlib/http2/__init__.py new file mode 100644 index 00000000..2803cccb --- /dev/null +++ b/netlib/http2/__init__.py @@ -0,0 +1,182 @@ +from __future__ import (absolute_import, print_function, division) +import itertools +import logging + +from .frame import * +from .. import utils + +log = logging.getLogger(__name__) + + +class HTTP2Protocol(object): + + ERROR_CODES = utils.BiDi( + NO_ERROR=0x0, + PROTOCOL_ERROR=0x1, + INTERNAL_ERROR=0x2, + FLOW_CONTROL_ERROR=0x3, + SETTINGS_TIMEOUT=0x4, + STREAM_CLOSED=0x5, + FRAME_SIZE_ERROR=0x6, + REFUSED_STREAM=0x7, + CANCEL=0x8, + COMPRESSION_ERROR=0x9, + CONNECT_ERROR=0xa, + ENHANCE_YOUR_CALM=0xb, + INADEQUATE_SECURITY=0xc, + HTTP_1_1_REQUIRED=0xd + ) + + # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a' + + ALPN_PROTO_H2 = 'h2' + + HTTP2_DEFAULT_SETTINGS = { + SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096, + SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1, + SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None, + SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ** 16 - 1, + SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ** 14, + SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None, + } + + def __init__(self, tcp_client): + self.tcp_client = tcp_client + + self.http2_settings = self.HTTP2_DEFAULT_SETTINGS.copy() + self.current_stream_id = None + self.encoder = Encoder() + self.decoder = Decoder() + + def check_alpn(self): + alp = self.tcp_client.get_alpn_proto_negotiated() + if alp != self.ALPN_PROTO_H2: + raise NotImplementedError( + "HTTP2Protocol can not handle unknown ALP: %s" % alp) + log.debug("ALP 'h2' successfully negotiated.") + return True + + def perform_connection_preface(self): + self.tcp_client.wfile.write( + bytes(self.CLIENT_CONNECTION_PREFACE.decode('hex'))) + self.send_frame(SettingsFrame(state=self)) + + # read server settings frame + frame = Frame.from_file(self.tcp_client.rfile, self) + assert isinstance(frame, SettingsFrame) + self._apply_settings(frame.settings) + + # read setting ACK frame + settings_ack_frame = self.read_frame() + assert isinstance(settings_ack_frame, SettingsFrame) + assert settings_ack_frame.flags & Frame.FLAG_ACK + assert len(settings_ack_frame.settings) == 0 + + log.debug("Connection Preface completed.") + + def next_stream_id(self): + if self.current_stream_id is None: + self.current_stream_id = 1 + else: + self.current_stream_id += 2 + return self.current_stream_id + + def send_frame(self, frame): + raw_bytes = frame.to_bytes() + self.tcp_client.wfile.write(raw_bytes) + self.tcp_client.wfile.flush() + + def read_frame(self): + frame = Frame.from_file(self.tcp_client.rfile, self) + if isinstance(frame, SettingsFrame): + self._apply_settings(frame.settings) + + return frame + + def _apply_settings(self, settings): + for setting, value in settings.items(): + old_value = self.http2_settings[setting] + if not old_value: + old_value = '-' + + self.http2_settings[setting] = value + log.debug("Setting changed: %s to %s (was %s)" % ( + SettingsFrame.SETTINGS.get_name(setting), + str(value), + str(old_value))) + + self.send_frame(SettingsFrame(state=self, flags=Frame.FLAG_ACK)) + log.debug("New settings acknowledged.") + + def _create_headers(self, headers, stream_id, end_stream=True): + # TODO: implement max frame size checks and sending in chunks + + flags = Frame.FLAG_END_HEADERS + if end_stream: + flags |= Frame.FLAG_END_STREAM + + header_block_fragment = self.encoder.encode(headers) + + bytes = HeadersFrame( + state=self, + flags=flags, + stream_id=stream_id, + header_block_fragment=header_block_fragment).to_bytes() + return [bytes] + + def _create_body(self, body, stream_id): + if body is None or len(body) == 0: + return b'' + + # TODO: implement max frame size checks and sending in chunks + # TODO: implement flow-control window + + bytes = DataFrame( + state=self, + flags=Frame.FLAG_END_STREAM, + stream_id=stream_id, + payload=body).to_bytes() + return [bytes] + + def create_request(self, method, path, headers=None, body=None): + if headers is None: + headers = [] + + headers = [ + (b':method', bytes(method)), + (b':path', bytes(path)), + (b':scheme', b'https')] + headers + + stream_id = self.next_stream_id() + + return list(itertools.chain( + self._create_headers(headers, stream_id, end_stream=(body is None)), + self._create_body(body, stream_id))) + + def read_response(self): + header_block_fragment = b'' + body = b'' + + while True: + frame = self.read_frame() + if isinstance(frame, HeadersFrame): + header_block_fragment += frame.header_block_fragment + if frame.flags | Frame.FLAG_END_HEADERS: + break + + while True: + frame = self.read_frame() + if isinstance(frame, DataFrame): + body += frame.payload + if frame.flags | Frame.FLAG_END_STREAM: + break + + headers = {} + for header, value in self.decoder.decode(header_block_fragment): + headers[header] = value + + for header, value in headers.items(): + log.debug("%s: %s" % (header, value)) + + return headers[':status'], headers, body diff --git a/netlib/h2/frame.py b/netlib/http2/frame.py index 36456c46..1497380a 100644 --- a/netlib/h2/frame.py +++ b/netlib/http2/frame.py @@ -1,8 +1,15 @@ import struct +import logging +from functools import reduce from hpack.hpack import Encoder, Decoder from .. import utils -from functools import reduce + +log = logging.getLogger(__name__) + + +class FrameSizeError(Exception): + pass class Frame(object): @@ -20,18 +27,53 @@ class Frame(object): FLAG_PADDED = 0x8 FLAG_PRIORITY = 0x20 - def __init__(self, length, flags, stream_id): + def __init__( + self, + state=None, + length=0, + flags=FLAG_NO_FLAGS, + stream_id=0x0): valid_flags = reduce(lambda x, y: x | y, self.VALID_FLAGS, 0x0) if flags | valid_flags != valid_flags: raise ValueError('invalid flags detected.') + if state is None: + from . import HTTP2Protocol + + class State(object): + pass + + state = State() + state.http2_settings = HTTP2Protocol.HTTP2_DEFAULT_SETTINGS.copy() + state.encoder = Encoder() + state.decoder = Decoder() + + self.state = state + self.length = length self.type = self.TYPE self.flags = flags self.stream_id = stream_id @classmethod - def from_file(self, fp): + def _check_frame_size(self, length, state): + from . import HTTP2Protocol + + if state: + settings = state.http2_settings + else: + settings = HTTP2Protocol.HTTP2_DEFAULT_SETTINGS + + max_frame_size = settings[ + SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE] + + if length > max_frame_size: + raise FrameSizeError( + "Frame size exceeded: %d, but only %d allowed." % ( + length, max_frame_size)) + + @classmethod + def from_file(self, fp, state=None): """ read a HTTP/2 frame sent by a server or client fp is a "file like" object that could be backed by a network @@ -44,22 +86,22 @@ class Frame(object): flags = fields[3] stream_id = fields[4] - payload = fp.safe_read(length) - return FRAMES[fields[2]].from_bytes(length, flags, stream_id, payload) + self._check_frame_size(length, state) - @classmethod - def from_bytes(self, data): - fields = struct.unpack("!HBBBL", data[:9]) - length = (fields[0] << 8) + fields[1] - # type is already deducted from class - flags = fields[3] - stream_id = fields[4] - return FRAMES[fields[2]].from_bytes(length, flags, stream_id, data[9:]) + payload = fp.safe_read(length) + return FRAMES[fields[2]].from_bytes( + state, + length, + flags, + stream_id, + payload) def to_bytes(self): payload = self.payload_bytes() self.length = len(payload) + self._check_frame_size(self.length, self.state) + b = struct.pack('!HB', self.length & 0xFFFF00, self.length & 0x0000FF) b += struct.pack('!B', self.TYPE) b += struct.pack('!B', self.flags) @@ -96,18 +138,19 @@ class DataFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, payload=b'', pad_length=0): - super(DataFrame, self).__init__(length, flags, stream_id) + super(DataFrame, self).__init__(state, length, flags, stream_id) self.payload = payload self.pad_length = pad_length @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) if f.flags & self.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] @@ -146,45 +189,39 @@ class HeadersFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, - headers=None, + header_block_fragment=b'', pad_length=0, exclusive=False, stream_dependency=0x0, weight=0): - super(HeadersFrame, self).__init__(length, flags, stream_id) - - if headers is None: - headers = [] + super(HeadersFrame, self).__init__(state, length, flags, stream_id) - self.headers = headers + self.header_block_fragment = header_block_fragment self.pad_length = pad_length self.exclusive = exclusive self.stream_dependency = stream_dependency self.weight = weight @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) if f.flags & self.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] - header_block_fragment = payload[1:-f.pad_length] + f.header_block_fragment = payload[1:-f.pad_length] else: - header_block_fragment = payload[0:] + f.header_block_fragment = payload[0:] if f.flags & self.FLAG_PRIORITY: f.stream_dependency, f.weight = struct.unpack( - '!LB', header_block_fragment[ - :5]) + '!LB', f.header_block_fragment[:5]) f.exclusive = bool(f.stream_dependency >> 31) f.stream_dependency &= 0x7FFFFFFF - header_block_fragment = header_block_fragment[5:] - - for header, value in Decoder().decode(header_block_fragment): - f.headers.append((header, value)) + f.header_block_fragment = f.header_block_fragment[5:] return f @@ -201,7 +238,7 @@ class HeadersFrame(Frame): (int(self.exclusive) << 31) | self.stream_dependency, self.weight) - b += Encoder().encode(self.headers) + b += self.header_block_fragment if self.flags & self.FLAG_PADDED: b += b'\0' * self.pad_length @@ -219,11 +256,9 @@ class HeadersFrame(Frame): if self.flags & self.FLAG_PADDED: s.append("padding: %d" % self.pad_length) - if not self.headers: - s.append("headers: None") - else: - for header, value in self.headers: - s.append("%s: %s" % (header, value)) + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) return "\n".join(s) @@ -234,20 +269,21 @@ class PriorityFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, exclusive=False, stream_dependency=0x0, weight=0): - super(PriorityFrame, self).__init__(length, flags, stream_id) + super(PriorityFrame, self).__init__(state, length, flags, stream_id) self.exclusive = exclusive self.stream_dependency = stream_dependency self.weight = weight @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.stream_dependency, f.weight = struct.unpack('!LB', payload) f.exclusive = bool(f.stream_dependency >> 31) @@ -283,16 +319,17 @@ class RstStreamFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, error_code=0x0): - super(RstStreamFrame, self).__init__(length, flags, stream_id) + super(RstStreamFrame, self).__init__(state, length, flags, stream_id) self.error_code = error_code @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.error_code = struct.unpack('!L', payload)[0] return f @@ -322,11 +359,12 @@ class SettingsFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, settings=None): - super(SettingsFrame, self).__init__(length, flags, stream_id) + super(SettingsFrame, self).__init__(state, length, flags, stream_id) if settings is None: settings = {} @@ -334,8 +372,8 @@ class SettingsFrame(Frame): self.settings = settings @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) for i in xrange(0, len(payload), 6): identifier, value = struct.unpack("!HL", payload[i:i + 6]) @@ -372,20 +410,21 @@ class PushPromiseFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, promised_stream=0x0, header_block_fragment=b'', pad_length=0): - super(PushPromiseFrame, self).__init__(length, flags, stream_id) + super(PushPromiseFrame, self).__init__(state, length, flags, stream_id) self.pad_length = pad_length self.promised_stream = promised_stream self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) if f.flags & self.FLAG_PADDED: f.pad_length, f.promised_stream = struct.unpack('!BL', payload[:5]) @@ -435,16 +474,17 @@ class PingFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, payload=b''): - super(PingFrame, self).__init__(length, flags, stream_id) + super(PingFrame, self).__init__(state, length, flags, stream_id) self.payload = payload @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.payload = payload return f @@ -467,20 +507,21 @@ class GoAwayFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, last_stream=0x0, error_code=0x0, data=b''): - super(GoAwayFrame, self).__init__(length, flags, stream_id) + super(GoAwayFrame, self).__init__(state, length, flags, stream_id) self.last_stream = last_stream self.error_code = error_code self.data = data @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.last_stream, f.error_code = struct.unpack("!LL", payload[:8]) f.last_stream &= 0x7FFFFFFF @@ -511,16 +552,17 @@ class WindowUpdateFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, window_size_increment=0x0): - super(WindowUpdateFrame, self).__init__(length, flags, stream_id) + super(WindowUpdateFrame, self).__init__(state, length, flags, stream_id) self.window_size_increment = window_size_increment @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.window_size_increment = struct.unpack("!L", payload)[0] f.window_size_increment &= 0x7FFFFFFF @@ -544,16 +586,17 @@ class ContinuationFrame(Frame): def __init__( self, + state=None, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, header_block_fragment=b''): - super(ContinuationFrame, self).__init__(length, flags, stream_id) + super(ContinuationFrame, self).__init__(state, length, flags, stream_id) self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, length, flags, stream_id, payload): - f = self(length=length, flags=flags, stream_id=stream_id) + def from_bytes(self, state, length, flags, stream_id, payload): + f = self(state=state, length=length, flags=flags, stream_id=stream_id) f.header_block_fragment = payload return f diff --git a/netlib/test.py b/netlib/test.py index 14f50157..1e1b5e9d 100644 --- a/netlib/test.py +++ b/netlib/test.py @@ -4,6 +4,7 @@ import Queue import cStringIO import OpenSSL from . import tcp, certutils +from test import tutils class ServerThread(threading.Thread): @@ -55,22 +56,33 @@ class TServer(tcp.TCPServer): dhparams, v3_only """ tcp.TCPServer.__init__(self, addr) - self.ssl, self.q = ssl, q + + if ssl is True: + self.ssl = dict() + elif isinstance(ssl, dict): + self.ssl = ssl + else: + self.ssl = None + + self.q = q self.handler_klass = handler_klass self.last_handler = None def handle_client_connection(self, request, client_address): h = self.handler_klass(request, client_address, self) self.last_handler = h - if self.ssl: - cert = certutils.SSLCert.from_pem( - file(self.ssl["cert"], "rb").read() - ) - raw = file(self.ssl["key"], "rb").read() + if self.ssl is not None: + raw_cert = self.ssl.get( + "cert", + tutils.test_data.path("data/server.crt")) + cert = certutils.SSLCert.from_pem(open(raw_cert, "rb").read()) + raw_key = self.ssl.get( + "key", + tutils.test_data.path("data/server.key")) key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, - raw) - if self.ssl["v3_only"]: + open(raw_key, "rb").read()) + if self.ssl.get("v3_only", False): method = tcp.SSLv3_METHOD options = OpenSSL.SSL.OP_NO_SSLv2 | OpenSSL.SSL.OP_NO_TLSv1 else: @@ -81,7 +93,7 @@ class TServer(tcp.TCPServer): method=method, options=options, handle_sni=getattr(h, "handle_sni", None), - request_client_cert=self.ssl["request_client_cert"], + request_client_cert=self.ssl.get("request_client_cert", None), cipher_list=self.ssl.get("cipher_list", None), dhparams=self.ssl.get("dhparams", None), chain_file=self.ssl.get("chain_file", None), |