From 9ea68ebd284ce13d765519a20dd7cfe998c0ae1c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 14:34:09 +1200 Subject: Improve handling of pseudo-headers - The canonical source for :method, :scheme and :path are the .method, .scheme and .path attributes on the request object. - These pseudo-headers are stripped after reading the request, and re-inserted just before sending. - The :authority header remains, and should be handled analagously to the Host header in HTTP1 with respect to display and user interaction. --- test/netlib/http/http2/test_connections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/http2/test_connections.py b/test/netlib/http/http2/test_connections.py index ff462ba6..69667d1c 100644 --- a/test/netlib/http/http2/test_connections.py +++ b/test/netlib/http/http2/test_connections.py @@ -312,7 +312,10 @@ class TestReadRequest(tservers.ServerTestBase): req = protocol.read_request(NotImplemented) assert req.stream_id - assert req.headers.fields == ((b':method', b'GET'), (b':path', b'/'), (b':scheme', b'https')) + assert req.headers.fields == () + assert req.method == "GET" + assert req.path == "/" + assert req.scheme == "https" assert req.content == b'foobar' -- cgit v1.2.3 From 08fbe6f1118455bc44d05db30b83bdf81feda2a0 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 17:16:31 +1200 Subject: Start cleaning up netlib.utils - Remove http2 functions, move to http2.frame - Remove Serializable, move to netlib.basetypes --- test/netlib/http/http2/test_connections.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/http2/test_connections.py b/test/netlib/http/http2/test_connections.py index 69667d1c..be68a28c 100644 --- a/test/netlib/http/http2/test_connections.py +++ b/test/netlib/http/http2/test_connections.py @@ -1,12 +1,12 @@ import mock import codecs -from hyperframe import frame - -from netlib import tcp, http, utils +import hyperframe +from netlib import tcp, http from netlib.tutils import raises from netlib.exceptions import TcpDisconnect from netlib.http.http2.connections import HTTP2Protocol, TCPHandler +from netlib.http.http2 import frame from ... import tservers @@ -111,11 +111,11 @@ class TestPerformServerConnectionPreface(tservers.ServerTestBase): self.wfile.flush() # check empty settings frame - raw = utils.http2_read_raw_frame(self.rfile) + raw = frame.http2_read_raw_frame(self.rfile) assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec') # check settings acknowledgement - raw = utils.http2_read_raw_frame(self.rfile) + raw = frame.http2_read_raw_frame(self.rfile) assert raw == codecs.decode('000000040100000000', 'hex_codec') # send settings acknowledgement @@ -214,19 +214,19 @@ class TestApplySettings(tservers.ServerTestBase): protocol = HTTP2Protocol(c) protocol._apply_settings({ - frame.SettingsFrame.ENABLE_PUSH: 'foo', - frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar', - frame.SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef', + hyperframe.frame.SettingsFrame.ENABLE_PUSH: 'foo', + hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar', + hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef', }) assert c.rfile.safe_read(2) == b"OK" assert protocol.http2_settings[ - frame.SettingsFrame.ENABLE_PUSH] == 'foo' + hyperframe.frame.SettingsFrame.ENABLE_PUSH] == 'foo' assert protocol.http2_settings[ - frame.SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar' + hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar' assert protocol.http2_settings[ - frame.SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef' + hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef' class TestCreateHeaders(object): @@ -258,7 +258,7 @@ class TestCreateHeaders(object): (b'server', b'version')]) protocol = HTTP2Protocol(self.c) - protocol.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] = 8 + protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 8 bytes = protocol._create_headers(headers, 1, end_stream=True) assert len(bytes) == 3 assert bytes[0] == codecs.decode('000008010100000001828487408294e783', 'hex_codec') @@ -281,7 +281,7 @@ class TestCreateBody(object): def test_create_body_multiple_frames(self): protocol = HTTP2Protocol(self.c) - protocol.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] = 5 + protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 5 bytes = protocol._create_body(b'foobarmehm42', 1) assert len(bytes) == 3 assert bytes[0] == codecs.decode('000005000000000001666f6f6261', 'hex_codec') -- cgit v1.2.3 From 4e6c9c4e935458d23add259dc63c5e0a85fba9c8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 18:42:56 +1200 Subject: Extract url functions from netlib.utils and move to netlib.http.url --- test/netlib/http/test_url.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test/netlib/http/test_url.py (limited to 'test/netlib/http') diff --git a/test/netlib/http/test_url.py b/test/netlib/http/test_url.py new file mode 100644 index 00000000..d777a949 --- /dev/null +++ b/test/netlib/http/test_url.py @@ -0,0 +1,65 @@ +from netlib import tutils +from netlib.http import url + +def test_parse_url(): + with tutils.raises(ValueError): + url.parse_url("") + + s, h, po, pa = url.parse_url(b"http://foo.com:8888/test") + assert s == b"http" + assert h == b"foo.com" + assert po == 8888 + assert pa == b"/test" + + s, h, po, pa = url.parse_url("http://foo/bar") + assert s == b"http" + assert h == b"foo" + assert po == 80 + assert pa == b"/bar" + + s, h, po, pa = url.parse_url(b"http://user:pass@foo/bar") + assert s == b"http" + assert h == b"foo" + assert po == 80 + assert pa == b"/bar" + + s, h, po, pa = url.parse_url(b"http://foo") + assert pa == b"/" + + s, h, po, pa = url.parse_url(b"https://foo") + assert po == 443 + + with tutils.raises(ValueError): + url.parse_url(b"https://foo:bar") + + # Invalid IDNA + with tutils.raises(ValueError): + url.parse_url("http://\xfafoo") + # Invalid PATH + with tutils.raises(ValueError): + url.parse_url("http:/\xc6/localhost:56121") + # Null byte in host + with tutils.raises(ValueError): + url.parse_url("http://foo\0") + # Port out of range + _, _, port, _ = url.parse_url("http://foo:999999") + assert port == 80 + # Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt + with tutils.raises(ValueError): + url.parse_url('http://lo[calhost') + + +def test_unparse_url(): + assert url.unparse_url("http", "foo.com", 99, "") == "http://foo.com:99" + assert url.unparse_url("http", "foo.com", 80, "/bar") == "http://foo.com/bar" + assert url.unparse_url("https", "foo.com", 80, "") == "https://foo.com:80" + assert url.unparse_url("https", "foo.com", 443, "") == "https://foo.com" + + +def test_urlencode(): + assert url.urlencode([('foo', 'bar')]) + + +def test_urldecode(): + s = "one=two&three=four" + assert len(url.urldecode(s)) == 2 -- cgit v1.2.3 From 6dda2b2ee544c3890f04b7bf99272998e29992b6 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 18:46:19 +1200 Subject: Module is part of the name - url.decode, not url.urldecode A pattern we need to use far more often in the codebase --- test/netlib/http/test_url.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/test_url.py b/test/netlib/http/test_url.py index d777a949..697c83a8 100644 --- a/test/netlib/http/test_url.py +++ b/test/netlib/http/test_url.py @@ -1,65 +1,65 @@ from netlib import tutils from netlib.http import url -def test_parse_url(): +def test_parse(): with tutils.raises(ValueError): - url.parse_url("") + url.parse("") - s, h, po, pa = url.parse_url(b"http://foo.com:8888/test") + s, h, po, pa = url.parse(b"http://foo.com:8888/test") assert s == b"http" assert h == b"foo.com" assert po == 8888 assert pa == b"/test" - s, h, po, pa = url.parse_url("http://foo/bar") + s, h, po, pa = url.parse("http://foo/bar") assert s == b"http" assert h == b"foo" assert po == 80 assert pa == b"/bar" - s, h, po, pa = url.parse_url(b"http://user:pass@foo/bar") + s, h, po, pa = url.parse(b"http://user:pass@foo/bar") assert s == b"http" assert h == b"foo" assert po == 80 assert pa == b"/bar" - s, h, po, pa = url.parse_url(b"http://foo") + s, h, po, pa = url.parse(b"http://foo") assert pa == b"/" - s, h, po, pa = url.parse_url(b"https://foo") + s, h, po, pa = url.parse(b"https://foo") assert po == 443 with tutils.raises(ValueError): - url.parse_url(b"https://foo:bar") + url.parse(b"https://foo:bar") # Invalid IDNA with tutils.raises(ValueError): - url.parse_url("http://\xfafoo") + url.parse("http://\xfafoo") # Invalid PATH with tutils.raises(ValueError): - url.parse_url("http:/\xc6/localhost:56121") + url.parse("http:/\xc6/localhost:56121") # Null byte in host with tutils.raises(ValueError): - url.parse_url("http://foo\0") + url.parse("http://foo\0") # Port out of range - _, _, port, _ = url.parse_url("http://foo:999999") + _, _, port, _ = url.parse("http://foo:999999") assert port == 80 # Invalid IPv6 URL - see http://www.ietf.org/rfc/rfc2732.txt with tutils.raises(ValueError): - url.parse_url('http://lo[calhost') + url.parse('http://lo[calhost') -def test_unparse_url(): - assert url.unparse_url("http", "foo.com", 99, "") == "http://foo.com:99" - assert url.unparse_url("http", "foo.com", 80, "/bar") == "http://foo.com/bar" - assert url.unparse_url("https", "foo.com", 80, "") == "https://foo.com:80" - assert url.unparse_url("https", "foo.com", 443, "") == "https://foo.com" +def test_unparse(): + assert url.unparse("http", "foo.com", 99, "") == "http://foo.com:99" + assert url.unparse("http", "foo.com", 80, "/bar") == "http://foo.com/bar" + assert url.unparse("https", "foo.com", 80, "") == "https://foo.com:80" + assert url.unparse("https", "foo.com", 443, "") == "https://foo.com" def test_urlencode(): - assert url.urlencode([('foo', 'bar')]) + assert url.encode([('foo', 'bar')]) def test_urldecode(): s = "one=two&three=four" - assert len(url.urldecode(s)) == 2 + assert len(url.decode(s)) == 2 -- cgit v1.2.3 From 15b2374ef9d6a8cbafdff7c79694921387836ff3 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 18:54:42 +1200 Subject: netlib.utils.get_header_tokens -> netlib.http1.read.get_header_tokens Placing this next to its only use. --- test/netlib/http/http1/test_read.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/http1/test_read.py b/test/netlib/http/http1/test_read.py index 974aa895..5285ac1d 100644 --- a/test/netlib/http/http1/test_read.py +++ b/test/netlib/http/http1/test_read.py @@ -7,11 +7,22 @@ from netlib.http.http1.read import ( read_request, read_response, read_request_head, read_response_head, read_body, connection_close, expected_http_body_size, _get_first_line, _read_request_line, _parse_authority_form, _read_response_line, _check_http_version, - _read_headers, _read_chunked + _read_headers, _read_chunked, get_header_tokens ) from netlib.tutils import treq, tresp, raises +def test_get_header_tokens(): + headers = Headers() + assert get_header_tokens(headers, "foo") == [] + headers["foo"] = "bar" + assert get_header_tokens(headers, "foo") == ["bar"] + headers["foo"] = "bar, voing" + assert get_header_tokens(headers, "foo") == ["bar", "voing"] + headers.set_all("foo", ["bar, voing", "oink"]) + assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"] + + def test_read_request(): rfile = BytesIO(b"GET / HTTP/1.1\r\n\r\nskip") r = read_request(rfile) -- cgit v1.2.3 From ec34cae6181d6af0150ac730d70b96104a07e9d5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 19:07:55 +1200 Subject: utils.multipartdecode -> http.multipart.decode also utils.parse_content_type -> http.headers.parse_content_type --- test/netlib/http/test_headers.py | 10 ++++++++++ test/netlib/http/test_multipart.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 test/netlib/http/test_multipart.py (limited to 'test/netlib/http') diff --git a/test/netlib/http/test_headers.py b/test/netlib/http/test_headers.py index cd2ca9d1..e12bceaf 100644 --- a/test/netlib/http/test_headers.py +++ b/test/netlib/http/test_headers.py @@ -1,4 +1,5 @@ from netlib.http import Headers +from netlib.http.headers import parse_content_type from netlib.tutils import raises @@ -72,3 +73,12 @@ class TestHeaders(object): replacements = headers.replace(r"Host: ", "X-Host ") assert replacements == 0 assert headers["Host"] == "example.com" + + +def test_parse_content_type(): + p = parse_content_type + assert p("text/html") == ("text", "html", {}) + assert p("text") is None + + v = p("text/html; charset=UTF-8") + assert v == ('text', 'html', {'charset': 'UTF-8'}) diff --git a/test/netlib/http/test_multipart.py b/test/netlib/http/test_multipart.py new file mode 100644 index 00000000..45ae996b --- /dev/null +++ b/test/netlib/http/test_multipart.py @@ -0,0 +1,23 @@ +from netlib.http import Headers +from netlib.http import multipart + +def test_decode(): + boundary = 'somefancyboundary' + headers = Headers( + content_type='multipart/form-data; boundary=' + boundary + ) + content = ( + "--{0}\n" + "Content-Disposition: form-data; name=\"field1\"\n\n" + "value1\n" + "--{0}\n" + "Content-Disposition: form-data; name=\"field2\"\n\n" + "value2\n" + "--{0}--".format(boundary).encode() + ) + + form = multipart.decode(headers, content) + + assert len(form) == 2 + assert form[0] == (b"field1", b"value1") + assert form[1] == (b"field2", b"value2") -- cgit v1.2.3 From 40a030f215e1943aefdb2eb6fe2a264b9b1ee33c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 19:58:28 +1200 Subject: Satisfy flake8 --- test/netlib/http/test_multipart.py | 1 + test/netlib/http/test_url.py | 1 + 2 files changed, 2 insertions(+) (limited to 'test/netlib/http') diff --git a/test/netlib/http/test_multipart.py b/test/netlib/http/test_multipart.py index 45ae996b..1d7e0062 100644 --- a/test/netlib/http/test_multipart.py +++ b/test/netlib/http/test_multipart.py @@ -1,6 +1,7 @@ from netlib.http import Headers from netlib.http import multipart + def test_decode(): boundary = 'somefancyboundary' headers = Headers( diff --git a/test/netlib/http/test_url.py b/test/netlib/http/test_url.py index 697c83a8..26b37230 100644 --- a/test/netlib/http/test_url.py +++ b/test/netlib/http/test_url.py @@ -1,6 +1,7 @@ from netlib import tutils from netlib.http import url + def test_parse(): with tutils.raises(ValueError): url.parse("") -- cgit v1.2.3 From 5a75ea3fc65d08c802b4d5fea73e1494ce90aa7d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 20:19:54 +1200 Subject: Fix test failures --- test/netlib/http/test_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/test_response.py b/test/netlib/http/test_response.py index 1faef7ec..b3c2f736 100644 --- a/test/netlib/http/test_response.py +++ b/test/netlib/http/test_response.py @@ -24,7 +24,7 @@ class TestResponseCore(object): """ def test_repr(self): response = tresp() - assert repr(response) == "Response(200 OK, unknown content type, 7B)" + assert repr(response) == "Response(200 OK, unknown content type, 7b)" response.content = None assert repr(response) == "Response(200 OK, no content)" -- cgit v1.2.3 From 42e91fcfe1d27ca989c75a7939f652fdfcc47604 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 31 May 2016 20:32:28 +1200 Subject: http2.frame -> http2.framereader --- test/netlib/http/http2/test_connections.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test/netlib/http') diff --git a/test/netlib/http/http2/test_connections.py b/test/netlib/http/http2/test_connections.py index be68a28c..27cc30ba 100644 --- a/test/netlib/http/http2/test_connections.py +++ b/test/netlib/http/http2/test_connections.py @@ -6,7 +6,7 @@ from netlib import tcp, http from netlib.tutils import raises from netlib.exceptions import TcpDisconnect from netlib.http.http2.connections import HTTP2Protocol, TCPHandler -from netlib.http.http2 import frame +from netlib.http.http2 import framereader from ... import tservers @@ -111,11 +111,11 @@ class TestPerformServerConnectionPreface(tservers.ServerTestBase): self.wfile.flush() # check empty settings frame - raw = frame.http2_read_raw_frame(self.rfile) + raw = framereader.http2_read_raw_frame(self.rfile) assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec') # check settings acknowledgement - raw = frame.http2_read_raw_frame(self.rfile) + raw = framereader.http2_read_raw_frame(self.rfile) assert raw == codecs.decode('000000040100000000', 'hex_codec') # send settings acknowledgement -- cgit v1.2.3