diff options
Diffstat (limited to 'netlib')
| -rw-r--r-- | netlib/http/__init__.py | 4 | ||||
| -rw-r--r-- | netlib/http/message.py | 69 | ||||
| -rw-r--r-- | netlib/http/request.py | 59 | ||||
| -rw-r--r-- | netlib/http/response.py | 20 | ||||
| -rw-r--r-- | netlib/multidict.py | 49 | ||||
| -rw-r--r-- | netlib/odict.py | 81 |
6 files changed, 129 insertions, 153 deletions
diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index 9fafa28f..c4eb1d58 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division from .request import Request from .response import Response from .headers import Headers -from .message import MultiDictView, decoded +from .message import decoded from . import http1, http2, status_codes __all__ = [ "Request", "Response", "Headers", - "MultiDictView", "decoded", + "decoded", "http1", "http2", "status_codes", ] diff --git a/netlib/http/message.py b/netlib/http/message.py index 76affeec..028f43a1 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -242,72 +242,3 @@ class decoded(object): def __exit__(self, type, value, tb): if self.ce: self.message.encode(self.ce) - - -class MultiDictView(MultiDict): - """ - Some parts in HTTP (Cookies, URL query strings, ...) require a specific data structure: A MultiDict. - It behaves mostly like an ordered dict but it can have several values for the same key. - - The MultiDictView provides a MultiDict *view* on an :py:class:`Request` or :py:class:`Response`. - That is, it represents a part of the request as a MultiDict, but doesn't contain state/data themselves. - - For example, ``request.cookies`` provides a view on the ``Cookie: ...`` header. - Any change to ``request.cookies`` will also modify the ``Cookie`` header. - Any change to the ``Cookie`` header will also modify ``request.cookies``. - - Example: - - .. code-block:: python - - # Cookies are represented as a MultiDict. - >>> request.cookies - MultiDictView[("name", "value"), ("a", "false"), ("a", "42")] - - # MultiDicts mostly behave like a normal dict. - >>> request.cookies["name"] - "value" - - # If there is more than one value, only the first value is returned. - >>> request.cookies["a"] - "false" - - # `.get_all(key)` returns a list of all values. - >>> request.cookies.get_all("a") - ["false", "42"] - - # Changes to the headers are immediately reflected in the cookies. - >>> request.cookies - MultiDictView[("name", "value"), ...] - >>> del request.headers["Cookie"] - >>> request.cookies - MultiDictView[] # empty now - """ - - def __init__(self, attr, message): - if False: # pragma: no cover - # We do not want to call the parent constructor here as that - # would cause an unnecessary parse/unparse pass. - # This is here to silence linters. Message - super(MultiDictView, self).__init__(None) - self._attr = attr - self._message = message # type: Message - - @staticmethod - def _kconv(key): - # All request-attributes are case-sensitive. - return key - - @staticmethod - def _reduce_values(values): - # We just return the first element if - # multiple elements exist with the same key. - return values[0] - - @property - def fields(self): - return getattr(self._message, "_" + self._attr) - - @fields.setter - def fields(self, value): - setattr(self._message, self._attr, value) diff --git a/netlib/http/request.py b/netlib/http/request.py index ae28084b..056a2d93 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -10,8 +10,9 @@ from netlib import utils from netlib.http import cookies from netlib.odict import ODict from .. import encoding +from ..multidict import MultiDictView from .headers import Headers -from .message import Message, _native, _always_bytes, MessageData, MultiDictView +from .message import Message, _native, _always_bytes, MessageData # This regex extracts & splits the host header into host and port. # Handles the edge case of IPv6 addresses containing colons. @@ -228,20 +229,25 @@ class Request(Message): """ The request query string as an :py:class:`MultiDictView` object. """ - return MultiDictView("query", self) + return MultiDictView( + self._get_query, + self._set_query + ) - @property - def _query(self): + def _get_query(self): _, _, _, _, query, _ = urllib.parse.urlparse(self.url) return tuple(utils.urldecode(query)) - @query.setter - def query(self, value): + def _set_query(self, value): query = utils.urlencode(value) scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(self.url) _, _, _, self.path = utils.parse_url( urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment])) + @query.setter + def query(self, value): + self._set_query(value) + @property def cookies(self): # type: () -> MultiDictView @@ -250,16 +256,21 @@ class Request(Message): An empty :py:class:`MultiDictView` object if the cookie monster ate them all. """ - return MultiDictView("cookies", self) + return MultiDictView( + self._get_cookies, + self._set_cookies + ) - @property - def _cookies(self): + def _get_cookies(self): h = self.headers.get_all("Cookie") return tuple(cookies.parse_cookie_headers(h)) + def _set_cookies(self, value): + self.headers["cookie"] = cookies.format_cookie_header(value) + @cookies.setter def cookies(self, value): - self.headers["cookie"] = cookies.format_cookie_header(value) + self._set_cookies(value) @property def path_components(self): @@ -322,17 +333,18 @@ class Request(Message): An empty MultiDictView if the content-type indicates non-form data or the content could not be parsed. """ - return MultiDictView("urlencoded_form", self) + return MultiDictView( + self._get_urlencoded_form, + self._set_urlencoded_form + ) - @property - def _urlencoded_form(self): + def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: return tuple(utils.urldecode(self.content)) return () - @urlencoded_form.setter - def urlencoded_form(self, value): + def _set_urlencoded_form(self, value): """ Sets the body to the URL-encoded form data, and adds the appropriate content-type header. This will overwrite the existing content if there is one. @@ -340,21 +352,30 @@ class Request(Message): self.headers["content-type"] = "application/x-www-form-urlencoded" self.content = utils.urlencode(value) + @urlencoded_form.setter + def urlencoded_form(self, value): + self._set_urlencoded_form(value) + @property def multipart_form(self): """ The multipart form data as an :py:class:`MultipartFormDict` object. None if the content-type indicates non-form data. """ - return MultiDictView("multipart_form", self) + return MultiDictView( + self._get_multipart_form, + self._set_multipart_form + ) - @property - def _multipart_form(self): + def _get_multipart_form(self): is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() if is_valid_content_type: return utils.multipartdecode(self.headers, self.content) return () + def _set_multipart_form(self, value): + raise NotImplementedError() + @multipart_form.setter def multipart_form(self, value): - raise NotImplementedError() + self._set_multipart_form(value) diff --git a/netlib/http/response.py b/netlib/http/response.py index 6d56fc1f..7d272e10 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -5,7 +5,8 @@ import time from . import cookies from .headers import Headers -from .message import Message, _native, _always_bytes, MessageData, MultiDictView +from .message import Message, _native, _always_bytes, MessageData +from ..multidict import MultiDictView from .. import utils @@ -80,21 +81,26 @@ class Response(Message): Caveats: Updating the attr """ - return MultiDictView("cookies", self) + return MultiDictView( + self._get_cookies, + self._set_cookies + ) - @property - def _cookies(self): + def _get_cookies(self): h = self.headers.get_all("set-cookie") return tuple(cookies.parse_set_cookie_headers(h)) - @cookies.setter - def cookies(self, all_cookies): + def _set_cookies(self, value): cookie_headers = [] - for k, v in all_cookies: + for k, v in value: header = cookies.format_set_cookie_header(k, v[0], v[1]) cookie_headers.append(header) self.headers.set_all("set-cookie", cookie_headers) + @cookies.setter + def cookies(self, value): + self._set_cookies(value) + def refresh(self, now=None): """ This fairly complex and heuristic function refreshes a server diff --git a/netlib/multidict.py b/netlib/multidict.py index ec1b24d8..248acdec 100644 --- a/netlib/multidict.py +++ b/netlib/multidict.py @@ -15,13 +15,7 @@ from .utils import Serializable @six.add_metaclass(ABCMeta) -class MultiDict(MutableMapping, Serializable): - def __init__(self, fields=None): - - # it is important for us that .fields is immutable, so that we can easily - # detect changes to it. - self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...] - +class _MultiDict(MutableMapping, Serializable): def __repr__(self): fields = tuple( repr(field) @@ -100,7 +94,7 @@ class MultiDict(MutableMapping, Serializable): value for k, value in self.fields if self._kconv(k) == key - ] + ] def set_all(self, key, values): """ @@ -176,7 +170,7 @@ class MultiDict(MutableMapping, Serializable): if multi: return self.fields else: - return super(MultiDict, self).items() + return super(_MultiDict, self).items() def to_dict(self): """ @@ -216,6 +210,12 @@ class MultiDict(MutableMapping, Serializable): return cls(tuple(x) for x in state) +class MultiDict(_MultiDict): + def __init__(self, fields=None): + super(MultiDict, self).__init__() + self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...] + + @six.add_metaclass(ABCMeta) class ImmutableMultiDict(MultiDict): def _immutable(self, *_): @@ -249,3 +249,34 @@ class ImmutableMultiDict(MultiDict): ret = self.copy() super(ImmutableMultiDict, ret).insert(index, key, value) return ret + + +class MultiDictView(_MultiDict): + """ + The MultiDictView provides the MultiDict interface over calculated data. + The view itself contains no state - data is retrieved from the parent on + request, and stored back to the parent on change. + """ + def __init__(self, getter, setter): + self._getter = getter + self._setter = setter + super(MultiDictView, self).__init__() + + @staticmethod + def _kconv(key): + # All request-attributes are case-sensitive. + return key + + @staticmethod + def _reduce_values(values): + # We just return the first element if + # multiple elements exist with the same key. + return values[0] + + @property + def fields(self): + return self._getter() + + @fields.setter + def fields(self, value): + return self._setter(value) diff --git a/netlib/odict.py b/netlib/odict.py index 461192f7..8a638dab 100644 --- a/netlib/odict.py +++ b/netlib/odict.py @@ -1,5 +1,6 @@ from __future__ import (absolute_import, print_function, division) import copy + import six from .utils import Serializable, safe_subn @@ -27,27 +28,24 @@ class ODict(Serializable): def __iter__(self): return self.lst.__iter__() - def __getitem__(self, k): + def __getitem__(self, key): """ Returns a list of values matching key. """ - ret = [] - k = self._kconv(k) - for i in self.lst: - if self._kconv(i[0]) == k: - ret.append(i[1]) - return ret - def keys(self): - return list(set([self._kconv(i[0]) for i in self.lst])) + key = self._kconv(key) + return [ + v + for k, v in self.lst + if self._kconv(k) == key + ] - def _filter_lst(self, k, lst): - k = self._kconv(k) - new = [] - for i in lst: - if self._kconv(i[0]) != k: - new.append(i) - return new + def keys(self): + return list( + set( + self._kconv(k) for k, _ in self.lst + ) + ) def __len__(self): """ @@ -81,14 +79,19 @@ class ODict(Serializable): """ Delete all items matching k. """ - self.lst = self._filter_lst(k, self.lst) - - def __contains__(self, k): k = self._kconv(k) - for i in self.lst: - if self._kconv(i[0]) == k: - return True - return False + self.lst = [ + i + for i in self.lst + if self._kconv(i[0]) != k + ] + + def __contains__(self, key): + key = self._kconv(key) + return any( + self._kconv(k) == key + for k, _ in self.lst + ) def add(self, key, value, prepend=False): if prepend: @@ -127,40 +130,24 @@ class ODict(Serializable): def __repr__(self): return repr(self.lst) - def in_any(self, key, value, caseless=False): - """ - Do any of the values matching key contain value? - - If caseless is true, value comparison is case-insensitive. - """ - if caseless: - value = value.lower() - for i in self[key]: - if caseless: - i = i.lower() - if value in i: - return True - return False - def replace(self, pattern, repl, *args, **kwargs): """ Replaces a regular expression pattern with repl in both keys and - values. Encoded content will be decoded before replacement, and - re-encoded afterwards. + values. Returns the number of replacements made. """ - nlst, count = [], 0 - for i in self.lst: - k, c = safe_subn(pattern, repl, i[0], *args, **kwargs) + new, count = [], 0 + for k, v in self.lst: + k, c = safe_subn(pattern, repl, k, *args, **kwargs) count += c - v, c = safe_subn(pattern, repl, i[1], *args, **kwargs) + v, c = safe_subn(pattern, repl, v, *args, **kwargs) count += c - nlst.append([k, v]) - self.lst = nlst + new.append([k, v]) + self.lst = new return count - # Implement the StateObject protocol from mitmproxy + # Implement Serializable def get_state(self): return [tuple(i) for i in self.lst] |