Start cleaning up netlib.utils

- Remove http2 functions, move to http2.frame
- Remove Serializable, move to netlib.basetypes

4 files changed, 60 insertions, 39 deletions

diff --git a/netlib/http/http2/connections.py b/netlib/http/http2/connections.py
index 6b91f2ff..03f1804b 100644
--- a/netlib/http/http2/connections.py
+++ b/netlib/http/http2/connections.py
@@ -2,11 +2,12 @@ from __future__ import (absolute_import, print_function, division)
 import itertools
 import time
 
+import hyperframe.frame
+
 from hpack.hpack import Encoder, Decoder
 from ... import utils
 from .. import Headers, Response, Request
-
-from hyperframe import frame
+from . import frame
 
 
 class TCPHandler(object):
@@ -38,12 +39,12 @@ class HTTP2Protocol(object):
     CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'
 
     HTTP2_DEFAULT_SETTINGS = {
-        frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
-        frame.SettingsFrame.ENABLE_PUSH: 1,
-        frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None,
-        frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
-        frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14,
-        frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None,
+        hyperframe.frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
+        hyperframe.frame.SettingsFrame.ENABLE_PUSH: 1,
+        hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None,
+        hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
+        hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14,
+        hyperframe.frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None,
     }
 
     def __init__(
@@ -253,9 +254,9 @@ class HTTP2Protocol(object):
             magic = self.tcp_handler.rfile.safe_read(magic_length)
             assert magic == self.CLIENT_CONNECTION_PREFACE
 
-            frm = frame.SettingsFrame(settings={
-                frame.SettingsFrame.ENABLE_PUSH: 0,
-                frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1,
+            frm = hyperframe.frame.SettingsFrame(settings={
+                hyperframe.frame.SettingsFrame.ENABLE_PUSH: 0,
+                hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1,
             })
             self.send_frame(frm, hide=True)
             self._receive_settings(hide=True)
@@ -266,7 +267,7 @@ class HTTP2Protocol(object):
 
             self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE)
 
-            self.send_frame(frame.SettingsFrame(), hide=True)
+            self.send_frame(hyperframe.frame.SettingsFrame(), hide=True)
             self._receive_settings(hide=True)  # server announces own settings
             self._receive_settings(hide=True)  # server acks my settings
 
@@ -279,18 +280,18 @@ class HTTP2Protocol(object):
 
     def read_frame(self, hide=False):
         while True:
-            frm = utils.http2_read_frame(self.tcp_handler.rfile)
+            frm = frame.http2_read_frame(self.tcp_handler.rfile)
             if not hide and self.dump_frames:  # pragma no cover
                 print(frm.human_readable("<<"))
 
-            if isinstance(frm, frame.PingFrame):
-                raw_bytes = frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize()
+            if isinstance(frm, hyperframe.frame.PingFrame):
+                raw_bytes = hyperframe.frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize()
                 self.tcp_handler.wfile.write(raw_bytes)
                 self.tcp_handler.wfile.flush()
                 continue
-            if isinstance(frm, frame.SettingsFrame) and 'ACK' not in frm.flags:
+            if isinstance(frm, hyperframe.frame.SettingsFrame) and 'ACK' not in frm.flags:
                 self._apply_settings(frm.settings, hide)
-            if isinstance(frm, frame.DataFrame) and frm.flow_controlled_length > 0:
+            if isinstance(frm, hyperframe.frame.DataFrame) and frm.flow_controlled_length > 0:
                 self._update_flow_control_window(frm.stream_id, frm.flow_controlled_length)
             return frm
 
@@ -302,7 +303,7 @@ class HTTP2Protocol(object):
         return True
 
     def _handle_unexpected_frame(self, frm):
-        if isinstance(frm, frame.SettingsFrame):
+        if isinstance(frm, hyperframe.frame.SettingsFrame):
             return
         if self.unhandled_frame_cb:
             self.unhandled_frame_cb(frm)
@@ -310,7 +311,7 @@ class HTTP2Protocol(object):
     def _receive_settings(self, hide=False):
         while True:
             frm = self.read_frame(hide)
-            if isinstance(frm, frame.SettingsFrame):
+            if isinstance(frm, hyperframe.frame.SettingsFrame):
                 break
             else:
                 self._handle_unexpected_frame(frm)
@@ -334,26 +335,26 @@ class HTTP2Protocol(object):
                 old_value = '-'
             self.http2_settings[setting] = value
 
-        frm = frame.SettingsFrame(flags=['ACK'])
+        frm = hyperframe.frame.SettingsFrame(flags=['ACK'])
         self.send_frame(frm, hide)
 
     def _update_flow_control_window(self, stream_id, increment):
-        frm = frame.WindowUpdateFrame(stream_id=0, window_increment=increment)
+        frm = hyperframe.frame.WindowUpdateFrame(stream_id=0, window_increment=increment)
         self.send_frame(frm)
-        frm = frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment)
+        frm = hyperframe.frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment)
         self.send_frame(frm)
 
     def _create_headers(self, headers, stream_id, end_stream=True):
         def frame_cls(chunks):
             for i in chunks:
                 if i == 0:
-                    yield frame.HeadersFrame, i
+                    yield hyperframe.frame.HeadersFrame, i
                 else:
-                    yield frame.ContinuationFrame, i
+                    yield hyperframe.frame.ContinuationFrame, i
 
         header_block_fragment = self.encoder.encode(headers.fields)
 
-        chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE]
+        chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
         chunks = range(0, len(header_block_fragment), chunk_size)
         frms = [frm_cls(
             flags=[],
@@ -374,9 +375,9 @@ class HTTP2Protocol(object):
         if body is None or len(body) == 0:
             return b''
 
-        chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE]
+        chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
         chunks = range(0, len(body), chunk_size)
-        frms = [frame.DataFrame(
+        frms = [hyperframe.frame.DataFrame(
             flags=[],
             stream_id=stream_id,
             data=body[i:i + chunk_size]) for i in chunks]
@@ -400,7 +401,7 @@ class HTTP2Protocol(object):
         while True:
             frm = self.read_frame()
             if (
-                (isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame)) and
+                (isinstance(frm, hyperframe.frame.HeadersFrame) or isinstance(frm, hyperframe.frame.ContinuationFrame)) and
                 (stream_id is None or frm.stream_id == stream_id)
             ):
                 stream_id = frm.stream_id
@@ -414,7 +415,7 @@ class HTTP2Protocol(object):
 
         while body_expected:
             frm = self.read_frame()
-            if isinstance(frm, frame.DataFrame) and frm.stream_id == stream_id:
+            if isinstance(frm, hyperframe.frame.DataFrame) and frm.stream_id == stream_id:
                 body += frm.data
                 if 'END_STREAM' in frm.flags:
                     break
diff --git a/netlib/http/http2/frame.py b/netlib/http/http2/frame.py
new file mode 100644
index 00000000..d45be646
--- /dev/null
+++ b/netlib/http/http2/frame.py
@@ -0,0 +1,21 @@
+import codecs
+
+import hyperframe
+
+
+def http2_read_raw_frame(rfile):
+    header = rfile.safe_read(9)
+    length = int(codecs.encode(header[:3], 'hex_codec'), 16)
+
+    if length == 4740180:
+        raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20))
+
+    body = rfile.safe_read(length)
+    return [header, body]
+
+
+def http2_read_frame(rfile):
+    header, body = http2_read_raw_frame(rfile)
+    frame, length = hyperframe.frame.Frame.parse_frame_header(header)
+    frame.parse_body(memoryview(body))
+    return frame
diff --git a/netlib/http/message.py b/netlib/http/message.py
index 13d401a7..d9654f26 100644
--- a/netlib/http/message.py
+++ b/netlib/http/message.py
@@ -4,9 +4,8 @@ import warnings
 
 import six
 
-from .headers import Headers
-from .. import encoding, utils
-from ..utils import always_bytes
+from .. import encoding, utils, basetypes
+from . import headers
 
 if six.PY2:  # pragma: no cover
     def _native(x):
@@ -20,10 +19,10 @@ else:
         return x.decode("utf-8", "surrogateescape")
 
     def _always_bytes(x):
-        return always_bytes(x, "utf-8", "surrogateescape")
+        return utils.always_bytes(x, "utf-8", "surrogateescape")
 
 
-class MessageData(utils.Serializable):
+class MessageData(basetypes.Serializable):
     def __eq__(self, other):
         if isinstance(other, MessageData):
             return self.__dict__ == other.__dict__
@@ -38,7 +37,7 @@ class MessageData(utils.Serializable):
     def set_state(self, state):
         for k, v in state.items():
             if k == "headers":
-                v = Headers.from_state(v)
+                v = headers.Headers.from_state(v)
             setattr(self, k, v)
 
     def get_state(self):
@@ -48,11 +47,11 @@ class MessageData(utils.Serializable):
 
     @classmethod
     def from_state(cls, state):
-        state["headers"] = Headers.from_state(state["headers"])
+        state["headers"] = headers.Headers.from_state(state["headers"])
         return cls(**state)
 
 
-class Message(utils.Serializable):
+class Message(basetypes.Serializable):
     def __eq__(self, other):
         if isinstance(other, Message):
             return self.data == other.data
@@ -72,7 +71,7 @@ class Message(utils.Serializable):
 
     @classmethod
     def from_state(cls, state):
-        state["headers"] = Headers.from_state(state["headers"])
+        state["headers"] = headers.Headers.from_state(state["headers"])
         return cls(**state)
 
     @property
diff --git a/netlib/http/request.py b/netlib/http/request.py
index fa8d54aa..80a9ae65 100644
--- a/netlib/http/request.py
+++ b/netlib/http/request.py
@@ -6,7 +6,7 @@ import six
 from six.moves import urllib
 
 from netlib import utils
-from netlib.http import cookies
+from . import cookies
 from .. import encoding
 from ..multidict import MultiDictView
 from .headers import Headers