diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/console/common.py | 4 | ||||
| -rw-r--r-- | libmproxy/console/contentview.py | 14 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 19 | ||||
| -rw-r--r-- | libmproxy/dump.py | 2 | ||||
| -rw-r--r-- | libmproxy/filt.py | 42 | ||||
| -rw-r--r-- | libmproxy/flow.py | 34 | ||||
| -rw-r--r-- | libmproxy/models/http.py | 62 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 8 | ||||
| -rw-r--r-- | libmproxy/web/app.py | 7 |
9 files changed, 99 insertions, 93 deletions
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index c25f7267..ae3dd61e 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -415,9 +415,9 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2, resp_clen = contentdesc, roundtrip = roundtrip, )) - t = f.response.headers["content-type"] + t = f.response.headers.get("content-type") if t: - d["resp_ctype"] = t[0].split(";")[0] + d["resp_ctype"] = t.split(";")[0] else: d["resp_ctype"] = "" return flowcache.get( diff --git a/libmproxy/console/contentview.py b/libmproxy/console/contentview.py index 95ea7b17..17ed90e1 100644 --- a/libmproxy/console/contentview.py +++ b/libmproxy/console/contentview.py @@ -12,7 +12,7 @@ import urwid import html2text import netlib.utils -from netlib import odict, encoding +from netlib import encoding from . import common, signals from .. import utils @@ -74,7 +74,7 @@ class ViewAuto: content_types = [] def __call__(self, hdrs, content, limit): - ctype = hdrs.get_first("content-type") + ctype = hdrs.get("content-type") if ctype: ct = netlib.utils.parse_content_type(ctype) if ctype else None ct = "%s/%s" % (ct[0], ct[1]) @@ -508,7 +508,7 @@ def get(name): return i -def get_content_view(viewmode, hdrItems, content, limit, is_request): +def get_content_view(viewmode, headers, content, limit, is_request): """ Returns a (msg, body) tuple. """ @@ -519,16 +519,14 @@ def get_content_view(viewmode, hdrItems, content, limit, is_request): return "No content", "" msg = [] - hdrs = odict.ODictCaseless([list(i) for i in hdrItems]) - - enc = hdrs.get_first("content-encoding") + enc = headers.get("content-encoding") if enc and enc != "identity": decoded = encoding.decode(enc, content) if decoded: content = decoded msg.append("[decoded %s]" % enc) try: - ret = viewmode(hdrs, content, limit) + ret = viewmode(headers, content, limit) # Third-party viewers can fail in unexpected ways... except Exception: s = traceback.format_exc() @@ -536,7 +534,7 @@ def get_content_view(viewmode, hdrItems, content, limit, is_request): signals.add_event(s, "error") ret = None if not ret: - ret = get("Raw")(hdrs, content, limit) + ret = get("Raw")(headers, content, limit) msg.append("Couldn't parse: falling back to Raw") else: msg.append(ret[0]) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 8b828653..19917555 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -4,7 +4,7 @@ import sys import urwid from netlib import odict -from netlib.http.semantics import CONTENT_MISSING +from netlib.http.semantics import CONTENT_MISSING, Headers from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview @@ -182,7 +182,7 @@ class FlowView(tabs.Tabs): description, text_objects = cache.get( contentview.get_content_view, viewmode, - tuple(tuple(i) for i in conn.headers.lst), + conn.headers, conn.content, limit, isinstance(conn, HTTPRequest) @@ -199,7 +199,7 @@ class FlowView(tabs.Tabs): def conn_text(self, conn): if conn: txt = common.format_keyvals( - [(h + ":", v) for (h, v) in conn.headers.lst], + [(h + ":", v) for (h, v) in conn.headers.fields], key = "header", val = "text" ) @@ -284,8 +284,8 @@ class FlowView(tabs.Tabs): response.msg = msg signals.flow_change.send(self, flow = self.flow) - def set_headers(self, lst, conn): - conn.headers = odict.ODictCaseless(lst) + def set_headers(self, fields, conn): + conn.headers = Headers(fields) signals.flow_change.send(self, flow = self.flow) def set_query(self, lst, conn): @@ -330,7 +330,7 @@ class FlowView(tabs.Tabs): if not self.flow.response: self.flow.response = HTTPResponse( self.flow.request.httpversion, - 200, "OK", odict.ODictCaseless(), "" + 200, "OK", Headers(), "" ) self.flow.response.reply = controller.DummyReply() message = self.flow.response @@ -381,7 +381,7 @@ class FlowView(tabs.Tabs): self.master.view_grideditor( grideditor.HeaderEditor( self.master, - message.headers.lst, + message.headers.fields, self.set_headers, message ) @@ -616,8 +616,7 @@ class FlowView(tabs.Tabs): key = None elif key == "v": if conn.content: - t = conn.headers["content-type"] or [None] - t = t[0] + t = conn.headers.get("content-type") if "EDITOR" in os.environ or "PAGER" in os.environ: self.master.spawn_external_viewer(conn.content, t) else: @@ -626,7 +625,7 @@ class FlowView(tabs.Tabs): ) elif key == "z": self.flow.backup() - e = conn.headers.get_first("content-encoding", "identity") + e = conn.headers.get("content-encoding", "identity") if e != "identity": if not conn.decode(): signals.status_message.send( diff --git a/libmproxy/dump.py b/libmproxy/dump.py index bf409803..17b47dd2 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -174,7 +174,7 @@ class DumpMaster(flow.FlowMaster): def _print_message(self, message): if self.o.flow_detail >= 2: - print(self.indent(4, message.headers.format()), file=self.outfile) + print(self.indent(4, str(message.headers)), file=self.outfile) if self.o.flow_detail >= 3: if message.content == CONTENT_MISSING: print(self.indent(4, "(content missing)"), file=self.outfile) diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 6abc4a11..7cd0f4df 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -77,17 +77,19 @@ class FResp(_Action): class _Rex(_Action): + flags = 0 + def __init__(self, expr): self.expr = expr try: - self.re = re.compile(self.expr) + self.re = re.compile(self.expr, self.flags) except: raise ValueError("Cannot compile expression.") def _check_content_type(expr, o): - val = o.headers["content-type"] - if val and re.search(expr, val[0]): + val = o.headers.get("content-type") + if val and re.search(expr, val): return True return False @@ -145,11 +147,12 @@ class FResponseContentType(_Rex): class FHead(_Rex): code = "h" help = "Header" + flags = re.MULTILINE def __call__(self, f): - if f.request.headers.match_re(self.expr): + if f.request and self.re.search(str(f.request.headers)): return True - elif f.response and f.response.headers.match_re(self.expr): + if f.response and self.re.search(str(f.response.headers)): return True return False @@ -157,18 +160,20 @@ class FHead(_Rex): class FHeadRequest(_Rex): code = "hq" help = "Request header" + flags = re.MULTILINE def __call__(self, f): - if f.request.headers.match_re(self.expr): + if f.request and self.re.search(str(f.request.headers)): return True class FHeadResponse(_Rex): code = "hs" help = "Response header" + flags = re.MULTILINE def __call__(self, f): - if f.response and f.response.headers.match_re(self.expr): + if f.response and self.re.search(str(f.response.headers)): return True @@ -178,10 +183,10 @@ class FBod(_Rex): def __call__(self, f): if f.request and f.request.content: - if re.search(self.expr, f.request.get_decoded_content()): + if self.re.search(f.request.get_decoded_content()): return True if f.response and f.response.content: - if re.search(self.expr, f.response.get_decoded_content()): + if self.re.search(f.response.get_decoded_content()): return True return False @@ -192,7 +197,7 @@ class FBodRequest(_Rex): def __call__(self, f): if f.request and f.request.content: - if re.search(self.expr, f.request.get_decoded_content()): + if self.re.search(f.request.get_decoded_content()): return True @@ -202,24 +207,26 @@ class FBodResponse(_Rex): def __call__(self, f): if f.response and f.response.content: - if re.search(self.expr, f.response.get_decoded_content()): + if self.re.search(f.response.get_decoded_content()): return True class FMethod(_Rex): code = "m" help = "Method" + flags = re.IGNORECASE def __call__(self, f): - return bool(re.search(self.expr, f.request.method, re.IGNORECASE)) + return bool(self.re.search(f.request.method)) class FDomain(_Rex): code = "d" help = "Domain" + flags = re.IGNORECASE def __call__(self, f): - return bool(re.search(self.expr, f.request.host, re.IGNORECASE)) + return bool(self.re.search(f.request.host)) class FUrl(_Rex): @@ -234,21 +241,24 @@ class FUrl(_Rex): return klass(*toks) def __call__(self, f): - return re.search(self.expr, f.request.url) + return self.re.search(f.request.url) + class FSrc(_Rex): code = "src" help = "Match source address" def __call__(self, f): - return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and self.re.search(repr(f.client_conn.address)) + class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and self.re.search(repr(f.server_conn.address)) + class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 5eac8da9..547d0f60 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,8 +11,8 @@ import re import urlparse -from netlib import odict, wsgi -from netlib.http.semantics import CONTENT_MISSING +from netlib import wsgi +from netlib.http.semantics import CONTENT_MISSING, Headers import netlib.http from . import controller, tnetstring, filt, script, version from .onboarding import app @@ -45,7 +45,7 @@ class AppRegistry: if (request.host, request.port) in self.apps: return self.apps[(request.host, request.port)] if "host" in request.headers: - host = request.headers["host"][0] + host = request.headers["host"] return self.apps.get((host, request.port), None) @@ -144,15 +144,15 @@ class SetHeaders: for _, header, value, cpatt in self.lst: if cpatt(f): if f.response: - del f.response.headers[header] + f.response.headers.pop(header, None) else: - del f.request.headers[header] + f.request.headers.pop(header, None) for _, header, value, cpatt in self.lst: if cpatt(f): if f.response: - f.response.headers.add(header, value) + f.response.headers.fields.append((header, value)) else: - f.request.headers.add(header, value) + f.request.headers.fields.append((header, value)) class StreamLargeBodies(object): @@ -278,14 +278,11 @@ class ServerPlaybackState: key.append(p[1]) if self.headers: - hdrs = [] + headers = [] for i in self.headers: - v = r.headers[i] - # Slightly subtle: we need to convert everything to strings - # to prevent a mismatch between unicode/non-unicode. - v = [str(x) for x in v] - hdrs.append((i, v)) - key.append(hdrs) + v = r.headers.get(i) + headers.append((i, v)) + key.append(headers) return hashlib.sha256(repr(key)).digest() def next_flow(self, request): @@ -329,7 +326,7 @@ class StickyCookieState: return False def handle_response(self, f): - for i in f.response.headers["set-cookie"]: + for i in f.response.headers.get_all("set-cookie"): # FIXME: We now know that Cookie.py screws up some cookies with # valid RFC 822/1123 datetime specifications for expiry. Sigh. c = Cookie.SimpleCookie(str(i)) @@ -351,7 +348,7 @@ class StickyCookieState: l.append(self.jar[i].output(header="").strip()) if l: f.request.stickycookie = True - f.request.headers["cookie"] = l + f.request.headers.set_all("cookie",l) class StickyAuthState: @@ -836,7 +833,7 @@ class FlowMaster(controller.Master): ssl_established=True )) f = HTTPFlow(c, s) - headers = odict.ODictCaseless() + headers = Headers() req = HTTPRequest( "absolute", @@ -930,8 +927,7 @@ class FlowMaster(controller.Master): f.backup() f.request.is_replay = True if f.request.content: - f.request.headers[ - "Content-Length"] = [str(len(f.request.content))] + f.request.headers["Content-Length"] = str(len(f.request.content)) f.response = None f.error = None self.process_new_request(f) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py index fb2f305b..0d5e53b5 100644 --- a/libmproxy/models/http.py +++ b/libmproxy/models/http.py @@ -5,8 +5,8 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import time from libmproxy import utils -from netlib import odict, encoding -from netlib.http import status_codes +from netlib import encoding +from netlib.http import status_codes, Headers from netlib.tcp import Address from netlib.http.semantics import Request, Response, CONTENT_MISSING from .. import version, stateobject @@ -16,7 +16,7 @@ from .flow import Flow class MessageMixin(stateobject.StateObject): _stateobject_attributes = dict( httpversion=tuple, - headers=odict.ODictCaseless, + headers=Headers, body=str, timestamp_start=float, timestamp_end=float @@ -40,7 +40,7 @@ class MessageMixin(stateobject.StateObject): header. Doesn't change the message iteself or its headers. """ - ce = self.headers.get_first("content-encoding") + ce = self.headers.get("content-encoding") if not self.body or ce not in encoding.ENCODINGS: return self.body return encoding.decode(ce, self.body) @@ -53,14 +53,14 @@ class MessageMixin(stateobject.StateObject): Returns True if decoding succeeded, False otherwise. """ - ce = self.headers.get_first("content-encoding") + ce = self.headers.get("content-encoding") if not self.body or ce not in encoding.ENCODINGS: return False data = encoding.decode(ce, self.body) if data is None: return False self.body = data - del self.headers["content-encoding"] + self.headers.pop("content-encoding", None) return True def encode(self, e): @@ -70,7 +70,7 @@ class MessageMixin(stateobject.StateObject): """ # FIXME: Error if there's an existing encoding header? self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] + self.headers["content-encoding"] = e def copy(self): c = copy.copy(self) @@ -86,11 +86,18 @@ class MessageMixin(stateobject.StateObject): Returns the number of replacements made. """ with decoded(self): - self.body, c = utils.safe_subn( + self.body, count = utils.safe_subn( pattern, repl, self.body, *args, **kwargs ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c + fields = [] + for name, value in self.headers.fields: + name, c = utils.safe_subn(pattern, repl, name, *args, **kwargs) + count += c + value, c = utils.safe_subn(pattern, repl, value, *args, **kwargs) + count += c + fields.append([name, value]) + self.headers.fields = fields + return count class HTTPRequest(MessageMixin, Request): @@ -115,7 +122,7 @@ class HTTPRequest(MessageMixin, Request): httpversion: HTTP version tuple, e.g. (1,1) - headers: odict.ODictCaseless object + headers: Headers object content: Content of the request, None, or CONTENT_MISSING if there is content associated, but not present. CONTENT_MISSING evaluates @@ -266,7 +273,7 @@ class HTTPResponse(MessageMixin, Response): msg: HTTP response message - headers: ODict Caseless object + headers: Headers object content: Content of the request, None, or CONTENT_MISSING if there is content associated, but not present. CONTENT_MISSING evaluates @@ -379,15 +386,15 @@ class HTTPResponse(MessageMixin, Response): ] for i in refresh_headers: if i in self.headers: - d = parsedate_tz(self.headers[i][0]) + d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] + self.headers[i] = formatdate(new) c = [] - for i in self.headers["set-cookie"]: + for i in self.headers.get_all("set-cookie"): c.append(self._refresh_cookie(i, delta)) if c: - self.headers["set-cookie"] = c + self.headers.set_all("set-cookie", c) class HTTPFlow(Flow): @@ -490,7 +497,7 @@ class decoded(object): def __init__(self, o): self.o = o - ce = o.headers.get_first("content-encoding") + ce = o.headers.get("content-encoding") if ce in encoding.ENCODINGS: self.ce = ce else: @@ -517,11 +524,12 @@ def make_error_response(status_code, message, headers=None): """.strip() % (status_code, response, message) if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] + headers = Headers( + Server=version.NAMEVERSION, + Connection="close", + Content_Length=str(len(body)), + Content_Type="text/html" + ) return HTTPResponse( (1, 1), # FIXME: Should be a string. @@ -536,15 +544,15 @@ def make_connect_request(address): address = Address.wrap(address) return HTTPRequest( "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" + Headers(), "" ) def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) + headers = Headers( + Content_Length="0", + Proxy_Agent=version.NAMEVERSION + ) return HTTPResponse( httpversion, 200, diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f51fea95..fbf4ac9b 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, print_function, division) from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address @@ -568,10 +568,6 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) + Headers(**self.config.authenticator.auth_challenge_headers()) )) raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py index d6082ee2..2517e7ad 100644 --- a/libmproxy/web/app.py +++ b/libmproxy/web/app.py @@ -27,8 +27,7 @@ class RequestHandler(tornado.web.RequestHandler): @property def json(self): - if not self.request.headers.get( - "Content-Type").startswith("application/json"): + if not self.request.headers.get("Content-Type").startswith("application/json"): return None return json.loads(self.request.body) @@ -186,12 +185,12 @@ class FlowContent(RequestHandler): if not message.content: raise APIError(400, "No content.") - content_encoding = message.headers.get_first("Content-Encoding", None) + content_encoding = message.headers.get("Content-Encoding", None) if content_encoding: content_encoding = re.sub(r"[^\w]", "", content_encoding) self.set_header("Content-Encoding", content_encoding) - original_cd = message.headers.get_first("Content-Disposition", None) + original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: filename = re.search("filename=([\w\" \.\-\(\)]+)", original_cd) |