aboutsummaryrefslogtreecommitdiffstats
path: root/netlib
diff options
context:
space:
mode:
Diffstat (limited to 'netlib')
-rw-r--r--netlib/http/authentication.py2
-rw-r--r--netlib/http/exceptions.py3
-rw-r--r--netlib/http/http1/protocol.py68
-rw-r--r--netlib/http/http2/frame.py2
-rw-r--r--netlib/http/http2/protocol.py27
-rw-r--r--netlib/http/semantics.py65
-rw-r--r--netlib/odict.py3
-rw-r--r--netlib/tutils.py6
-rw-r--r--netlib/utils.py61
-rw-r--r--netlib/websockets/frame.py6
-rw-r--r--netlib/websockets/protocol.py7
11 files changed, 156 insertions, 94 deletions
diff --git a/netlib/http/authentication.py b/netlib/http/authentication.py
index 9a227010..29b9eb3c 100644
--- a/netlib/http/authentication.py
+++ b/netlib/http/authentication.py
@@ -2,7 +2,6 @@ from __future__ import (absolute_import, print_function, division)
from argparse import Action, ArgumentTypeError
import binascii
-from .. import http
def parse_http_basic_auth(s):
words = s.split()
@@ -37,7 +36,6 @@ class NullProxyAuth(object):
"""
Clean up authentication headers, so they're not passed upstream.
"""
- pass
def authenticate(self, headers_):
"""
diff --git a/netlib/http/exceptions.py b/netlib/http/exceptions.py
index 7cd26c12..987a7908 100644
--- a/netlib/http/exceptions.py
+++ b/netlib/http/exceptions.py
@@ -1,6 +1,8 @@
from netlib import odict
+
class HttpError(Exception):
+
def __init__(self, code, message):
super(HttpError, self).__init__(message)
self.code = code
@@ -11,6 +13,7 @@ class HttpErrorConnClosed(HttpError):
class HttpAuthenticationError(Exception):
+
def __init__(self, auth_headers=None):
super(HttpAuthenticationError, self).__init__(
"Proxy Authentication Required"
diff --git a/netlib/http/http1/protocol.py b/netlib/http/http1/protocol.py
index 2e85a762..8eeb7744 100644
--- a/netlib/http/http1/protocol.py
+++ b/netlib/http/http1/protocol.py
@@ -1,28 +1,31 @@
from __future__ import (absolute_import, print_function, division)
-import binascii
-import collections
import string
import sys
-import urlparse
import time
from netlib import odict, utils, tcp, http
from netlib.http import semantics
-from .. import status_codes
from ..exceptions import *
+
class TCPHandler(object):
+
def __init__(self, rfile, wfile=None):
self.rfile = rfile
self.wfile = wfile
+
class HTTP1Protocol(semantics.ProtocolMixin):
def __init__(self, tcp_handler=None, rfile=None, wfile=None):
self.tcp_handler = tcp_handler or TCPHandler(rfile, wfile)
-
- def read_request(self, include_body=True, body_size_limit=None, allow_empty=False):
+ def read_request(
+ self,
+ include_body=True,
+ body_size_limit=None,
+ allow_empty=False,
+ ):
"""
Parse an HTTP request from a file stream
@@ -129,8 +132,12 @@ class HTTP1Protocol(semantics.ProtocolMixin):
timestamp_end,
)
-
- def read_response(self, request_method, body_size_limit, include_body=True):
+ def read_response(
+ self,
+ request_method,
+ body_size_limit,
+ include_body=True,
+ ):
"""
Returns an http.Response
@@ -175,7 +182,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
# read separately
body = None
-
if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = self.tcp_handler.rfile.first_byte_timestamp
@@ -195,7 +201,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
timestamp_end=timestamp_end,
)
-
def assemble_request(self, request):
assert isinstance(request, semantics.Request)
@@ -208,7 +213,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
headers = self._assemble_request_headers(request)
return "%s\r\n%s\r\n%s" % (first_line, headers, request.body)
-
def assemble_response(self, response):
assert isinstance(response, semantics.Response)
@@ -221,7 +225,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
headers = self._assemble_response_headers(response)
return "%s\r\n%s\r\n%s" % (first_line, headers, response.body)
-
def read_headers(self):
"""
Read a set of headers.
@@ -266,7 +269,7 @@ class HTTP1Protocol(semantics.ProtocolMixin):
response_code,
is_request,
max_chunk_size=None
- ):
+ ):
"""
Read an HTTP message body:
headers: An ODictCaseless object
@@ -321,9 +324,14 @@ class HTTP1Protocol(semantics.ProtocolMixin):
"HTTP Body too large. Limit is %s," % limit
)
-
@classmethod
- def expected_http_body_size(self, headers, is_request, request_method, response_code):
+ def expected_http_body_size(
+ self,
+ headers,
+ is_request,
+ request_method,
+ response_code,
+ ):
"""
Returns the expected body length:
- a positive integer, if the size is known in advance
@@ -360,20 +368,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
@classmethod
- def request_preamble(self, method, resource, http_major="1", http_minor="1"):
- return '%s %s HTTP/%s.%s' % (
- method, resource, http_major, http_minor
- )
-
-
- @classmethod
- def response_preamble(self, code, message=None, http_major="1", http_minor="1"):
- if message is None:
- message = status_codes.RESPONSES.get(code)
- return 'HTTP/%s.%s %s %s' % (http_major, http_minor, code, message)
-
-
- @classmethod
def has_chunked_encoding(self, headers):
return "chunked" in [
i.lower() for i in utils.get_header_tokens(headers, "transfer-encoding")
@@ -390,7 +384,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
line = self.tcp_handler.rfile.readline()
return line
-
def _read_chunked(self, limit, is_request):
"""
Read a chunked HTTP body.
@@ -427,7 +420,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
if length == 0:
return
-
@classmethod
def _parse_http_protocol(self, line):
"""
@@ -447,7 +439,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return None
return major, minor
-
@classmethod
def _parse_init(self, line):
try:
@@ -461,7 +452,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return None
return method, url, httpversion
-
@classmethod
def _parse_init_connect(self, line):
"""
@@ -489,7 +479,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return None
return host, port, httpversion
-
@classmethod
def _parse_init_proxy(self, line):
v = self._parse_init(line)
@@ -503,7 +492,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
scheme, host, port, path = parts
return method, scheme, host, port, path, httpversion
-
@classmethod
def _parse_init_http(self, line):
"""
@@ -519,7 +507,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return None
return method, url, httpversion
-
@classmethod
def connection_close(self, httpversion, headers):
"""
@@ -539,7 +526,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
# be persistent
return httpversion != (1, 1)
-
@classmethod
def parse_response_line(self, line):
parts = line.strip().split(" ", 2)
@@ -554,7 +540,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return None
return (proto, code, msg)
-
@classmethod
def _assemble_request_first_line(self, request):
return request.legacy_first_line()
@@ -575,7 +560,6 @@ class HTTP1Protocol(semantics.ProtocolMixin):
return headers.format()
-
def _assemble_response_first_line(self, response):
return 'HTTP/%s.%s %s %s' % (
response.httpversion[0],
@@ -584,7 +568,11 @@ class HTTP1Protocol(semantics.ProtocolMixin):
response.msg,
)
- def _assemble_response_headers(self, response, preserve_transfer_encoding=False):
+ def _assemble_response_headers(
+ self,
+ response,
+ preserve_transfer_encoding=False,
+ ):
headers = response.headers.copy()
for k in response._headers_to_strip_off:
del headers[k]
diff --git a/netlib/http/http2/frame.py b/netlib/http/http2/frame.py
index f7e60471..aa1fbae4 100644
--- a/netlib/http/http2/frame.py
+++ b/netlib/http/http2/frame.py
@@ -117,7 +117,7 @@ class Frame(object):
return "\n".join([
"%s: %s | length: %d | flags: %#x | stream_id: %d" % (
- direction, self.__class__.__name__, self.length, self.flags, self.stream_id),
+ direction, self.__class__.__name__, self.length, self.flags, self.stream_id),
self.payload_human_readable(),
"===============================================================",
])
diff --git a/netlib/http/http2/protocol.py b/netlib/http/http2/protocol.py
index a1ca4a18..c2ad5edd 100644
--- a/netlib/http/http2/protocol.py
+++ b/netlib/http/http2/protocol.py
@@ -9,6 +9,7 @@ from . import frame
class TCPHandler(object):
+
def __init__(self, rfile, wfile=None):
self.rfile = rfile
self.wfile = wfile
@@ -39,7 +40,6 @@ class HTTP2Protocol(semantics.ProtocolMixin):
ALPN_PROTO_H2 = 'h2'
-
def __init__(
self,
tcp_handler=None,
@@ -60,7 +60,12 @@ class HTTP2Protocol(semantics.ProtocolMixin):
self.current_stream_id = None
self.connection_preface_performed = False
- def read_request(self, include_body=True, body_size_limit=None, allow_empty=False):
+ def read_request(
+ self,
+ include_body=True,
+ body_size_limit=None,
+ allow_empty=False,
+ ):
self.perform_connection_preface()
timestamp_start = time.time()
@@ -92,7 +97,12 @@ class HTTP2Protocol(semantics.ProtocolMixin):
return request
- def read_response(self, request_method='', body_size_limit=None, include_body=True):
+ def read_response(
+ self,
+ request_method='',
+ body_size_limit=None,
+ include_body=True,
+ ):
self.perform_connection_preface()
timestamp_start = time.time()
@@ -123,7 +133,6 @@ class HTTP2Protocol(semantics.ProtocolMixin):
return response
-
def assemble_request(self, request):
assert isinstance(request, semantics.Request)
@@ -133,13 +142,13 @@ class HTTP2Protocol(semantics.ProtocolMixin):
headers = request.headers.copy()
- if not ':authority' in headers.keys():
+ if ':authority' not in headers.keys():
headers.add(':authority', bytes(authority), prepend=True)
- if not ':scheme' in headers.keys():
+ if ':scheme' not in headers.keys():
headers.add(':scheme', bytes(request.scheme), prepend=True)
- if not ':path' in headers.keys():
+ if ':path' not in headers.keys():
headers.add(':path', bytes(request.path), prepend=True)
- if not ':method' in headers.keys():
+ if ':method' not in headers.keys():
headers.add(':method', bytes(request.method), prepend=True)
headers = headers.items()
@@ -158,7 +167,7 @@ class HTTP2Protocol(semantics.ProtocolMixin):
headers = response.headers.copy()
- if not ':status' in headers.keys():
+ if ':status' not in headers.keys():
headers.add(':status', bytes(str(response.status_code)), prepend=True)
headers = headers.items()
diff --git a/netlib/http/semantics.py b/netlib/http/semantics.py
index e7ae2b5f..76213cd1 100644
--- a/netlib/http/semantics.py
+++ b/netlib/http/semantics.py
@@ -1,13 +1,9 @@
from __future__ import (absolute_import, print_function, division)
-import binascii
-import collections
-import string
-import sys
import urllib
import urlparse
from .. import utils, odict
-from . import cookies
+from . import cookies, exceptions
from netlib import utils, encoding
HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
@@ -18,11 +14,11 @@ CONTENT_MISSING = 0
class ProtocolMixin(object):
- def read_request(self):
- raise NotImplemented
+ def read_request(self, *args, **kwargs): # pragma: no cover
+ raise NotImplementedError
- def read_response(self):
- raise NotImplemented
+ def read_response(self, *args, **kwargs): # pragma: no cover
+ raise NotImplementedError
def assemble(self, message):
if isinstance(message, Request):
@@ -32,14 +28,23 @@ class ProtocolMixin(object):
else:
raise ValueError("HTTP message not supported.")
- def assemble_request(self, request):
- raise NotImplemented
+ def assemble_request(self, *args, **kwargs): # pragma: no cover
+ raise NotImplementedError
- def assemble_response(self, response):
- raise NotImplemented
+ def assemble_response(self, *args, **kwargs): # pragma: no cover
+ raise NotImplementedError
class Request(object):
+ # This list is adopted legacy code.
+ # We probably don't need to strip off keep-alive.
+ _headers_to_strip_off = [
+ 'Proxy-Connection',
+ 'Keep-Alive',
+ 'Connection',
+ 'Transfer-Encoding',
+ 'Upgrade',
+ ]
def __init__(
self,
@@ -71,7 +76,6 @@ class Request(object):
self.timestamp_start = timestamp_start
self.timestamp_end = timestamp_end
-
def __eq__(self, other):
try:
self_d = [self.__dict__[k] for k in self.__dict__ if k not in ('timestamp_start', 'timestamp_end')]
@@ -114,7 +118,7 @@ class Request(object):
self.httpversion[1],
)
else:
- raise http.HttpError(400, "Invalid request form")
+ raise exceptions.HttpError(400, "Invalid request form")
def anticache(self):
"""
@@ -143,7 +147,7 @@ class Request(object):
if self.headers["accept-encoding"]:
self.headers["accept-encoding"] = [
', '.join(
- e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0])]
+ e for e in encoding.ENCODINGS if e in self.headers.get_first("accept-encoding"))]
def update_host_header(self):
"""
@@ -317,17 +321,18 @@ class Request(object):
self.scheme, self.host, self.port, self.path = parts
@property
- def content(self):
+ def content(self): # pragma: no cover
# TODO: remove deprecated getter
return self.body
@content.setter
- def content(self, content):
+ def content(self, content): # pragma: no cover
# TODO: remove deprecated setter
self.body = content
class EmptyRequest(Request):
+
def __init__(self):
super(EmptyRequest, self).__init__(
form_in="",
@@ -339,10 +344,15 @@ class EmptyRequest(Request):
httpversion=(0, 0),
headers=odict.ODictCaseless(),
body="",
- )
+ )
class Response(object):
+ _headers_to_strip_off = [
+ 'Proxy-Connection',
+ 'Alternate-Protocol',
+ 'Alt-Svc',
+ ]
def __init__(
self,
@@ -368,7 +378,6 @@ class Response(object):
self.timestamp_start = timestamp_start
self.timestamp_end = timestamp_end
-
def __eq__(self, other):
try:
self_d = [self.__dict__[k] for k in self.__dict__ if k not in ('timestamp_start', 'timestamp_end')]
@@ -388,11 +397,9 @@ class Response(object):
status_code=self.status_code,
msg=self.msg,
contenttype=self.headers.get_first(
- "content-type", "unknown content type"
- ),
- size=size
- )
-
+ "content-type",
+ "unknown content type"),
+ size=size)
def get_cookies(self):
"""
@@ -430,21 +437,21 @@ class Response(object):
self.headers["Set-Cookie"] = values
@property
- def content(self):
+ def content(self): # pragma: no cover
# TODO: remove deprecated getter
return self.body
@content.setter
- def content(self, content):
+ def content(self, content): # pragma: no cover
# TODO: remove deprecated setter
self.body = content
@property
- def code(self):
+ def code(self): # pragma: no cover
# TODO: remove deprecated getter
return self.status_code
@code.setter
- def code(self, code):
+ def code(self, code): # pragma: no cover
# TODO: remove deprecated setter
self.status_code = code
diff --git a/netlib/odict.py b/netlib/odict.py
index d02de08d..11d5d52a 100644
--- a/netlib/odict.py
+++ b/netlib/odict.py
@@ -91,8 +91,9 @@ class ODict(object):
self.lst = self._filter_lst(k, self.lst)
def __contains__(self, k):
+ k = self._kconv(k)
for i in self.lst:
- if self._kconv(i[0]) == self._kconv(k):
+ if self._kconv(i[0]) == k:
return True
return False
diff --git a/netlib/tutils.py b/netlib/tutils.py
index 5018b9e8..7434c108 100644
--- a/netlib/tutils.py
+++ b/netlib/tutils.py
@@ -69,8 +69,6 @@ def raises(exc, obj, *args, **kwargs):
test_data = utils.Data(__name__)
-
-
def treq(content="content", scheme="http", host="address", port=22):
"""
@return: libmproxy.protocol.http.HTTPRequest
@@ -119,7 +117,7 @@ def tresp(content="message"):
"OK",
headers,
content,
- time.time(),
- time.time(),
+ timestamp_start=time.time(),
+ timestamp_end=time.time(),
)
return resp
diff --git a/netlib/utils.py b/netlib/utils.py
index 35ea0ec7..31dcd622 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -4,6 +4,7 @@ import cgi
import urllib
import urlparse
import string
+import re
def isascii(s):
@@ -118,6 +119,7 @@ def pretty_size(size):
class Data(object):
+
def __init__(self, name):
m = __import__(name)
dirname, _ = os.path.split(m.__file__)
@@ -136,8 +138,6 @@ class Data(object):
return fullpath
-
-
def is_valid_port(port):
if not 0 <= port <= 65535:
return False
@@ -220,6 +220,7 @@ def hostport(scheme, host, port):
else:
return "%s:%s" % (host, port)
+
def unparse_url(scheme, host, port, path=""):
"""
Returns a URL string, constructed from the specified compnents.
@@ -234,8 +235,64 @@ def urlencode(s):
s = [tuple(i) for i in s]
return urllib.urlencode(s, False)
+
def urldecode(s):
"""
Takes a urlencoded string and returns a list of (key, value) tuples.
"""
return cgi.parse_qsl(s, keep_blank_values=True)
+
+
+def parse_content_type(c):
+ """
+ A simple parser for content-type values. Returns a (type, subtype,
+ parameters) tuple, where type and subtype are strings, and parameters
+ is a dict. If the string could not be parsed, return None.
+
+ E.g. the following string:
+
+ text/html; charset=UTF-8
+
+ Returns:
+
+ ("text", "html", {"charset": "UTF-8"})
+ """
+ parts = c.split(";", 1)
+ ts = parts[0].split("/", 1)
+ if len(ts) != 2:
+ return None
+ d = {}
+ if len(parts) == 2:
+ for i in parts[1].split(";"):
+ clause = i.split("=", 1)
+ if len(clause) == 2:
+ d[clause[0].strip()] = clause[1].strip()
+ return ts[0].lower(), ts[1].lower(), d
+
+
+def multipartdecode(hdrs, content):
+ """
+ Takes a multipart boundary encoded string and returns list of (key, value) tuples.
+ """
+ v = hdrs.get_first("content-type")
+ if v:
+ v = parse_content_type(v)
+ if not v:
+ return []
+ boundary = v[2].get("boundary")
+ if not boundary:
+ return []
+
+ rx = re.compile(r'\bname="([^"]+)"')
+ r = []
+
+ for i in content.split("--" + boundary):
+ parts = i.splitlines()
+ if len(parts) > 1 and parts[0][0:2] != "--":
+ match = rx.search(parts[1])
+ if match:
+ key = match.group(1)
+ value = "".join(parts[3 + parts[2:].index(""):])
+ r.append((key, value))
+ return r
+ return []
diff --git a/netlib/websockets/frame.py b/netlib/websockets/frame.py
index 49d8ee10..1c4a03b2 100644
--- a/netlib/websockets/frame.py
+++ b/netlib/websockets/frame.py
@@ -1,12 +1,11 @@
from __future__ import absolute_import
-import base64
-import hashlib
import os
import struct
import io
from .protocol import Masker
-from netlib import utils, odict, tcp
+from netlib import tcp
+from netlib import utils
DEFAULT = object()
@@ -22,6 +21,7 @@ OPCODE = utils.BiDi(
PONG=0x0a
)
+
class FrameHeader(object):
def __init__(
diff --git a/netlib/websockets/protocol.py b/netlib/websockets/protocol.py
index 29b4db3d..6ce32eac 100644
--- a/netlib/websockets/protocol.py
+++ b/netlib/websockets/protocol.py
@@ -2,10 +2,9 @@ from __future__ import absolute_import
import base64
import hashlib
import os
-import struct
-import io
-from netlib import utils, odict, tcp
+from netlib import odict
+from netlib import utils
# Colleciton of utility functions that implement small portions of the RFC6455
# WebSockets Protocol Useful for building WebSocket clients and servers.
@@ -26,6 +25,7 @@ HEADER_WEBSOCKET_KEY = 'sec-websocket-key'
HEADER_WEBSOCKET_ACCEPT = 'sec-websocket-accept'
HEADER_WEBSOCKET_VERSION = 'sec-websocket-version'
+
class Masker(object):
"""
@@ -53,6 +53,7 @@ class Masker(object):
self.offset += len(ret)
return ret
+
class WebsocketsProtocol(object):
def __init__(self):
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-02 06:23:05 +0000