diff options
| -rw-r--r-- | netlib/http/cookies.py | 234 | ||||
| -rw-r--r-- | netlib/http/response.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/builtins/test_stickycookie.py | 1 | ||||
| -rw-r--r-- | test/netlib/http/test_cookies.py | 217 |
4 files changed, 269 insertions, 185 deletions
diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index 1421d8eb..a3ac4806 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -14,12 +14,9 @@ information. Duplicate cookies are preserved in parsing, and can be set in formatting. We do attempt to escape and quote values where needed, but will not reject data that violate the specs. -Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We do -not parse the comma-separated variant of Set-Cookie that allows multiple -cookies to be set in a single header. Technically this should be feasible, but -it turns out that violations of RFC6265 that makes the parsing problem -indeterminate are much more common than genuine occurences of the multi-cookie -variants. Serialization follows RFC6265. +Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We +also parse the comma-separated variant of Set-Cookie that allows multiple +cookies to be set in a single header. Serialization follows RFC6265. http://tools.ietf.org/html/rfc6265 http://tools.ietf.org/html/rfc2109 @@ -31,8 +28,21 @@ _cookie_params = set(( 'secure', 'httponly', 'version', )) +ESCAPE = re.compile(r"([\"\\])") + + +class CookieAttrs(multidict.ImmutableMultiDict): + @staticmethod + def _kconv(key): + return key.lower() -# TODO: Disallow LHS-only Cookie values + @staticmethod + def _reduce_values(values): + # See the StickyCookieTest for a weird cookie that only makes sense + # if we take the last part. + return values[-1] + +SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"]) def _read_until(s, start, term): @@ -47,13 +57,6 @@ def _read_until(s, start, term): return s[start:i + 1], i + 1 -def _read_token(s, start): - """ - Read a token - the LHS of a token/value pair in a cookie. - """ - return _read_until(s, start, ";=") - - def _read_quoted_string(s, start): """ start: offset to the first quote of the string to be read @@ -81,12 +84,16 @@ def _read_quoted_string(s, start): return "".join(ret), i + 1 +def _read_key(s, start, delims=";="): + """ + Read a key - the LHS of a token/value pair in a cookie. + """ + return _read_until(s, start, delims) + + def _read_value(s, start, delims): """ Reads a value - the RHS of a token/value pair in a cookie. - - special: If the value is special, commas are premitted. Else comma - terminates. This helps us support old and new style values. """ if start >= len(s): return "", start @@ -96,27 +103,82 @@ def _read_value(s, start, delims): return _read_until(s, start, delims) -def _read_pairs(s, off=0): +def _read_cookie_pairs(s, off=0): """ - Read pairs of lhs=rhs values. + Read pairs of lhs=rhs values from Cookie headers. off: start offset - specials: a lower-cased list of keys that may contain commas """ - vals = [] + pairs = [] + + while True: + lhs, off = _read_key(s, off) + lhs = lhs.lstrip() + + if lhs: + rhs = None + if off < len(s) and s[off] == "=": + rhs, off = _read_value(s, off + 1, ";") + + pairs.append([lhs, rhs]) + + off += 1 + + if not off < len(s): + break + + return pairs, off + + +def _read_set_cookie_pairs(s, off=0): + """ + Read pairs of lhs=rhs values from SetCookie headers while handling multiple cookies. + + off: start offset + specials: attributes that are treated specially + """ + cookies = [] + pairs = [] + while True: - lhs, off = _read_token(s, off) + lhs, off = _read_key(s, off, ";=,") lhs = lhs.lstrip() + if lhs: rhs = None - if off < len(s): - if s[off] == "=": - rhs, off = _read_value(s, off + 1, ";") - vals.append([lhs, rhs]) + if off < len(s) and s[off] == "=": + rhs, off = _read_value(s, off + 1, ";,") + + # Special handliing of attributes + if lhs.lower() == "expires": + # 'expires' values can contain commas in them so they need to + # be handled separately. + + # We actually bank on the fact that the expires value WILL + # contain a comma. Things will fail, if they don't. + + # '3' is just a heuristic we use to determine whether we've + # only read a part of the expires value and we should read more. + if len(rhs) <= 3: + trail, off = _read_value(s, off + 1, ";,") + rhs = rhs + "," + trail + + pairs.append([lhs, rhs]) + + # comma marks the beginning of a new cookie + if off < len(s) and s[off] == ",": + cookies.append(pairs) + pairs = [] + off += 1 + if not off < len(s): break - return vals, off + + if pairs or not cookies: + cookies.append(pairs) + + return cookies, off def _has_special(s): @@ -129,15 +191,12 @@ def _has_special(s): return False -ESCAPE = re.compile(r"([\"\\])") - - -def _format_pairs(lst, specials=(), sep="; "): +def _format_pairs(pairs, specials=(), sep="; "): """ specials: A lower-cased list of keys that will not be quoted. """ vals = [] - for k, v in lst: + for k, v in pairs: if v is None: vals.append(k) else: @@ -155,85 +214,78 @@ def _format_set_cookie_pairs(lst): ) -def _parse_set_cookie_pairs(s): +def parse_cookie_header(line): """ - For Set-Cookie, we support multiple cookies as described in RFC2109. - This function therefore returns a list of lists. + Parse a Cookie header value. + Returns a list of (lhs, rhs) tuples. """ - pairs, off_ = _read_pairs(s) + pairs, off_ = _read_cookie_pairs(line) return pairs -def parse_set_cookie_headers(headers): - ret = [] - for header in headers: - v = parse_set_cookie_header(header) - if v: - name, value, attrs = v - ret.append((name, SetCookie(value, attrs))) - return ret - - -class CookieAttrs(multidict.ImmutableMultiDict): - @staticmethod - def _kconv(key): - return key.lower() - - @staticmethod - def _reduce_values(values): - # See the StickyCookieTest for a weird cookie that only makes sense - # if we take the last part. - return values[-1] +def parse_cookie_headers(cookie_headers): + cookie_list = [] + for header in cookie_headers: + cookie_list.extend(parse_cookie_header(header)) + return cookie_list -SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"]) +def format_cookie_header(lst): + """ + Formats a Cookie header value. + """ + return _format_pairs(lst) def parse_set_cookie_header(line): """ Parse a Set-Cookie header value - Returns a (name, value, attrs) tuple, or None, where attrs is an - CookieAttrs dict of attributes. No attempt is made to parse attribute - values - they are treated purely as strings. + Returns a list of (name, value, attrs) tuple for each cokie, or None. + Where attrs is a CookieAttrs dict of attributes. No attempt is made + to parse attribute values - they are treated purely as strings. """ - pairs = _parse_set_cookie_pairs(line) - if pairs: - return pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:]) + cookie_pairs, off = _read_set_cookie_pairs(line) + cookies = [ + (pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:])) + for pairs in cookie_pairs if pairs + ] -def format_set_cookie_header(name, value, attrs): - """ - Formats a Set-Cookie header value. - """ - pairs = [(name, value)] - pairs.extend( - attrs.fields if hasattr(attrs, "fields") else attrs - ) - return _format_set_cookie_pairs(pairs) + if cookies: + return cookies + else: + return None -def parse_cookie_headers(cookie_headers): - cookie_list = [] - for header in cookie_headers: - cookie_list.extend(parse_cookie_header(header)) - return cookie_list +def parse_set_cookie_headers(headers): + rv = [] + for header in headers: + cookies = parse_set_cookie_header(header) + if cookies: + for name, value, attrs in cookies: + rv.append((name, SetCookie(value, attrs))) + return rv -def parse_cookie_header(line): +def format_set_cookie_header(set_cookies): """ - Parse a Cookie header value. - Returns a list of (lhs, rhs) tuples. + Formats a Set-Cookie header value. """ - pairs, off_ = _read_pairs(line) - return pairs + rv = [] -def format_cookie_header(lst): - """ - Formats a Cookie header value. - """ - return _format_pairs(lst) + for set_cookie in set_cookies: + name, value, attrs = set_cookie + + pairs = [(name, value)] + pairs.extend( + attrs.fields if hasattr(attrs, "fields") else attrs + ) + + rv.append(_format_set_cookie_pairs(pairs)) + + return ", ".join(rv) def refresh_set_cookie_header(c, delta): @@ -245,7 +297,7 @@ def refresh_set_cookie_header(c, delta): A refreshed Set-Cookie string """ - name, value, attrs = parse_set_cookie_header(c) + name, value, attrs = parse_set_cookie_header(c)[0] if not name or not value: raise ValueError("Invalid Cookie") @@ -263,10 +315,10 @@ def refresh_set_cookie_header(c, delta): # For now, we just ignore this. attrs = attrs.with_delitem("expires") - ret = format_set_cookie_header(name, value, attrs) - if not ret: + rv = format_set_cookie_header([(name, value, attrs)]) + if not rv: raise ValueError("Invalid Cookie") - return ret + return rv def get_expiration_ts(cookie_attrs): diff --git a/netlib/http/response.py b/netlib/http/response.py index 385e233a..ec19640d 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -155,7 +155,7 @@ class Response(message.Message): def _set_cookies(self, value): cookie_headers = [] for k, v in value: - header = cookies.format_set_cookie_header(k, v[0], v[1]) + header = cookies.format_set_cookie_header([(k, v[0], v[1])]) cookie_headers.append(header) self.headers.set_all("set-cookie", cookie_headers) diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py index ed92013d..4d3b7cb3 100644 --- a/test/mitmproxy/builtins/test_stickycookie.py +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -92,7 +92,6 @@ class TestStickyCookie(mastertest.MasterTest): "foo/bar=hello", "foo:bar=world", "foo@bar=fizz", - "foo,bar=buzz", ] for c in cs: f.response.headers["Set-Cookie"] = c diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py index efd8ba80..905f549c 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/netlib/http/test_cookies.py @@ -5,18 +5,62 @@ from netlib.tutils import raises import mock +cookie_pairs = [ + [ + "", + [] + ], + [ + "one=uno", + [["one", "uno"]] + ], + [ + "one", + [["one", None]] + ], + [ + "one=uno; two=due", + [["one", "uno"], ["two", "due"]] + ], + [ + 'one="uno"; two="\due"', + [["one", "uno"], ["two", "due"]] + ], + [ + 'one="un\\"o"', + [["one", 'un"o']] + ], + [ + 'one="uno,due"', + [["one", 'uno,due']] + ], + [ + "one=uno; two; three=tre", + [["one", "uno"], ["two", None], ["three", "tre"]] + ], + [ + "_lvs2=zHai1+Hq+Tc2vmc2r4GAbdOI5Jopg3EwsdUT9g=; " + "_rcc2=53VdltWl+Ov6ordflA==;", + [ + ["_lvs2", "zHai1+Hq+Tc2vmc2r4GAbdOI5Jopg3EwsdUT9g="], + ["_rcc2", "53VdltWl+Ov6ordflA=="] + ] + ] +] + -def test_read_token(): +def test_read_key(): tokens = [ [("foo", 0), ("foo", 3)], [("foo", 1), ("oo", 3)], + [(" foo", 0), (" foo", 4)], [(" foo", 1), ("foo", 4)], [(" foo;", 1), ("foo", 4)], [(" foo=", 1), ("foo", 4)], [(" foo=bar", 1), ("foo", 4)], ] for q, a in tokens: - assert cookies._read_token(*q) == a + assert cookies._read_key(*q) == a def test_read_quoted_string(): @@ -32,7 +76,7 @@ def test_read_quoted_string(): assert cookies._read_quoted_string(*q) == a -def test_read_pairs(): +def test_read_cookie_pairs(): vals = [ [ "one", @@ -64,134 +108,90 @@ def test_read_pairs(): ], ] for s, lst in vals: - ret, off = cookies._read_pairs(s) + ret, off = cookies._read_cookie_pairs(s) assert ret == lst def test_pairs_roundtrips(): - pairs = [ - [ - "", - [] - ], - [ - "one=uno", - [["one", "uno"]] - ], - [ - "one", - [["one", None]] - ], - [ - "one=uno; two=due", - [["one", "uno"], ["two", "due"]] - ], - [ - 'one="uno"; two="\due"', - [["one", "uno"], ["two", "due"]] - ], - [ - 'one="un\\"o"', - [["one", 'un"o']] - ], - [ - 'one="uno,due"', - [["one", 'uno,due']] - ], - [ - "one=uno; two; three=tre", - [["one", "uno"], ["two", None], ["three", "tre"]] - ], - [ - "_lvs2=zHai1+Hq+Tc2vmc2r4GAbdOI5Jopg3EwsdUT9g=; " - "_rcc2=53VdltWl+Ov6ordflA==;", - [ - ["_lvs2", "zHai1+Hq+Tc2vmc2r4GAbdOI5Jopg3EwsdUT9g="], - ["_rcc2", "53VdltWl+Ov6ordflA=="] - ] - ] - ] - for s, lst in pairs: - ret, off = cookies._read_pairs(s) - assert ret == lst - s2 = cookies._format_pairs(lst) - ret, off = cookies._read_pairs(s2) - assert ret == lst + for s, expected in cookie_pairs: + ret, off = cookies._read_cookie_pairs(s) + assert ret == expected + + s2 = cookies._format_pairs(expected) + ret, off = cookies._read_cookie_pairs(s2) + assert ret == expected def test_cookie_roundtrips(): - pairs = [ - [ - "one=uno", - [["one", "uno"]] - ], - [ - "one=uno; two=due", - [["one", "uno"], ["two", "due"]] - ], - ] - for s, lst in pairs: + for s, expected in cookie_pairs: ret = cookies.parse_cookie_header(s) - assert ret == lst - s2 = cookies.format_cookie_header(ret) + assert ret == expected + + s2 = cookies.format_cookie_header(expected) ret = cookies.parse_cookie_header(s2) - assert ret == lst + assert ret == expected def test_parse_set_cookie_pairs(): pairs = [ [ "one=uno", - [ + [[ ["one", "uno"] - ] + ]] ], [ "one=un\x20", - [ + [[ ["one", "un\x20"] - ] + ]] ], [ "one=uno; foo", - [ + [[ ["one", "uno"], ["foo", None] - ] + ]] ], [ "mun=1.390.f60; " "expires=sun, 11-oct-2015 12:38:31 gmt; path=/; " "domain=b.aol.com", - [ + [[ ["mun", "1.390.f60"], ["expires", "sun, 11-oct-2015 12:38:31 gmt"], ["path", "/"], ["domain", "b.aol.com"] - ] + ]] ], [ r'rpb=190%3d1%2616726%3d1%2634832%3d1%2634874%3d1; ' 'domain=.rubiconproject.com; ' 'expires=mon, 11-may-2015 21:54:57 gmt; ' 'path=/', - [ + [[ ['rpb', r'190%3d1%2616726%3d1%2634832%3d1%2634874%3d1'], ['domain', '.rubiconproject.com'], ['expires', 'mon, 11-may-2015 21:54:57 gmt'], ['path', '/'] - ] + ]] ], ] - for s, lst in pairs: - ret = cookies._parse_set_cookie_pairs(s) - assert ret == lst - s2 = cookies._format_set_cookie_pairs(ret) - ret2 = cookies._parse_set_cookie_pairs(s2) - assert ret2 == lst + for s, expected in pairs: + ret, off = cookies._read_set_cookie_pairs(s) + assert ret == expected + + s2 = cookies._format_set_cookie_pairs(expected[0]) + ret2, off = cookies._read_set_cookie_pairs(s2) + assert ret2 == expected def test_parse_set_cookie_header(): + def set_cookie_equal(obs, exp): + assert obs[0] == exp[0] + assert obs[1] == exp[1] + assert obs[2].items(multi=True) == exp[2] + vals = [ [ "", None @@ -201,28 +201,61 @@ def test_parse_set_cookie_header(): ], [ "one=uno", - ("one", "uno", ()) + [ + ("one", "uno", ()) + ] ], [ "one=uno; foo=bar", - ("one", "uno", (("foo", "bar"),)) + [ + ("one", "uno", (("foo", "bar"),)) + ] ], [ "one=uno; foo=bar; foo=baz", - ("one", "uno", (("foo", "bar"), ("foo", "baz"))) + [ + ("one", "uno", (("foo", "bar"), ("foo", "baz"))) + ] + ], + # Comma Separated Variant of Set-Cookie Headers + [ + "foo=bar, doo=dar", + [ + ("foo", "bar", ()), + ("doo", "dar", ()), + ] + ], + [ + "foo=bar; path=/, doo=dar; roo=rar; zoo=zar", + [ + ("foo", "bar", (("path", "/"),)), + ("doo", "dar", (("roo", "rar"), ("zoo", "zar"))), + ] + ], + [ + "foo=bar; expires=Mon, 24 Aug 2037", + [ + ("foo", "bar", (("expires", "Mon, 24 Aug 2037"),)), + ] + ], + [ + "foo=bar; expires=Mon, 24 Aug 2037 00:00:00 GMT, doo=dar", + [ + ("foo", "bar", (("expires", "Mon, 24 Aug 2037 00:00:00 GMT"),)), + ("doo", "dar", ()), + ] ], ] for s, expected in vals: ret = cookies.parse_set_cookie_header(s) if expected: - assert ret[0] == expected[0] - assert ret[1] == expected[1] - assert ret[2].items(multi=True) == expected[2] - s2 = cookies.format_set_cookie_header(*ret) + for i in range(len(expected)): + set_cookie_equal(ret[i], expected[i]) + + s2 = cookies.format_set_cookie_header(ret) ret2 = cookies.parse_set_cookie_header(s2) - assert ret2[0] == expected[0] - assert ret2[1] == expected[1] - assert ret2[2].items(multi=True) == expected[2] + for i in range(len(expected)): + set_cookie_equal(ret2[i], expected[i]) else: assert ret is None |