diff options
| -rw-r--r-- | libmproxy/console/flowview.py | 7 | ||||
| -rw-r--r-- | libmproxy/console/options.py | 3 | ||||
| -rw-r--r-- | libmproxy/contentview.py (renamed from libmproxy/console/contentview.py) | 246 | ||||
| -rw-r--r-- | test/test_console_contentview.py | 2 | ||||
| -rw-r--r-- | test/test_console_import.py | 5 |
5 files changed, 151 insertions, 112 deletions
diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 19917555..e33d4c43 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -6,9 +6,9 @@ import urwid from netlib import odict from netlib.http.semantics import CONTENT_MISSING, Headers -from . import common, grideditor, contentview, signals, searchable, tabs +from . import common, grideditor, signals, searchable, tabs from . import flowdetailview -from .. import utils, controller +from .. import utils, controller, contentview from ..models import HTTPRequest, HTTPResponse, decoded @@ -185,7 +185,8 @@ class FlowView(tabs.Tabs): conn.headers, conn.content, limit, - isinstance(conn, HTTPRequest) + isinstance(conn, HTTPRequest), + signals.add_event ) return (description, text_objects) diff --git a/libmproxy/console/options.py b/libmproxy/console/options.py index 58a4d469..0948e96d 100644 --- a/libmproxy/console/options.py +++ b/libmproxy/console/options.py @@ -1,6 +1,7 @@ import urwid -from . import common, signals, grideditor, contentview +from .. import contentview +from . import common, signals, grideditor from . import select, palettes footer = [ diff --git a/libmproxy/console/contentview.py b/libmproxy/contentview.py index 17ed90e1..a9b6cf95 100644 --- a/libmproxy/console/contentview.py +++ b/libmproxy/contentview.py @@ -2,23 +2,21 @@ from __future__ import absolute_import import cStringIO import json import logging +import subprocess +import traceback + import lxml.html import lxml.etree from PIL import Image from PIL.ExifTags import TAGS -import subprocess -import traceback -import urwid import html2text import netlib.utils +from . import utils +from .contrib import jsbeautifier +from .contrib.wbxml.ASCommandResponse import ASCommandResponse from netlib import encoding -from . import common, signals -from .. import utils -from ..contrib import jsbeautifier -from ..contrib.wbxml.ASCommandResponse import ASCommandResponse - try: import pyamf from pyamf import remoting, flex @@ -38,12 +36,54 @@ else: cssutils.ser.prefs.validOnly = False VIEW_CUTOFF = 1024 * 50 +KEY_MAX = 30 -def _view_text(content, total, limit): +def format_dict(d): """ - Generates a body for a chunk of text. + Transforms the given dictionary into a list of + ("key", key ) + ("value", value) + tuples, where key is padded to a uniform width. """ + max_key_len = max(len(k) for k in d.keys()) + max_key_len = min(max_key_len, KEY_MAX) + for key, value in d.items(): + key += ":" + key = key.ljust(max_key_len + 2) + yield ( + ("header", key), + ("text", value) + ) + + +def format_text(content, limit): + """ + Transforms the given content into + """ + content = netlib.utils.cleanBin(content) + + for line in content[:limit].splitlines(): + yield ("text", line) + + for msg in trailer(content, limit): + yield msg + + +def trailer(content, limit): + bytes_removed = len(content) - limit + if bytes_removed > 0: + yield ( + "cutoff", + "... {} of data not shown.".format(netlib.utils.pretty_size(bytes_removed)) + ) + + +""" +def _view_text(content, total, limit): + "" + Generates a body for a chunk of text. + "" txt = [] for i in netlib.utils.cleanBin(content).splitlines(): txt.append( @@ -66,9 +106,23 @@ def trailer(clen, txt, limit): ] ) ) +""" + +class View(object): + name = None + prompt = () + content_types = [] -class ViewAuto: + def __call__(self, hdrs, content, limit): + """ + Returns: + A (mode name, content generator) tuple. + """ + raise NotImplementedError() + + +class ViewAuto(View): name = "Auto" prompt = ("auto", "a") content_types = [] @@ -85,36 +139,36 @@ class ViewAuto: return get("Raw")(hdrs, content, limit) -class ViewRaw: +class ViewRaw(View): name = "Raw" prompt = ("raw", "r") content_types = [] def __call__(self, hdrs, content, limit): - txt = _view_text(content[:limit], len(content), limit) - return "Raw", txt + return "Raw", format_text(content, limit) -class ViewHex: +class ViewHex(View): name = "Hex" prompt = ("hex", "e") content_types = [] - def __call__(self, hdrs, content, limit): - txt = [] + @staticmethod + def _format(content, limit): for offset, hexa, s in netlib.utils.hexdump(content[:limit]): - txt.append(urwid.Text([ - ("offset", offset), - " ", - ("text", hexa), - " ", + yield ( + ("offset", offset + " "), + ("text", hexa + " "), ("text", s), - ])) - trailer(len(content), txt, limit) - return "Hex", txt + ) + for msg in trailer(content, limit): + yield msg + + def __call__(self, hdrs, content, limit): + return "Hex", self._format(content, limit) -class ViewXML: +class ViewXML(View): name = "XML" prompt = ("xml", "x") content_types = ["text/xml"] @@ -150,40 +204,23 @@ class ViewXML: pretty_print=True, xml_declaration=True, doctype=doctype or None, - encoding = docinfo.encoding + encoding=docinfo.encoding ) - txt = [] - for i in s[:limit].strip().split("\n"): - txt.append( - urwid.Text(("text", i)), - ) - trailer(len(content), txt, limit) - return "XML-like data", txt + return "XML-like data", format_text(s, limit) -class ViewJSON: +class ViewJSON(View): name = "JSON" prompt = ("json", "s") content_types = ["application/json"] def __call__(self, hdrs, content, limit): - lines = utils.pretty_json(content) - if lines: - txt = [] - sofar = 0 - for i in lines: - sofar += len(i) - txt.append( - urwid.Text(("text", i)), - ) - if sofar > limit: - break - trailer(sum(len(i) for i in lines), txt, limit) - return "JSON", txt + pretty_json = utils.pretty_json(content) + return "JSON", format_text(pretty_json, limit) -class ViewHTML: +class ViewHTML(View): name = "HTML" prompt = ("html", "h") content_types = ["text/html"] @@ -201,10 +238,10 @@ class ViewHTML: pretty_print=True, doctype=docinfo.doctype ) - return "HTML", _view_text(s[:limit], len(s), limit) + return "HTML", format_text(s, limit) -class ViewHTMLOutline: +class ViewHTMLOutline(View): name = "HTML Outline" prompt = ("html outline", "o") content_types = ["text/html"] @@ -215,43 +252,34 @@ class ViewHTMLOutline: h.ignore_images = True h.body_width = 0 content = h.handle(content) - txt = _view_text(content[:limit], len(content), limit) - return "HTML Outline", txt + return "HTML Outline", format_text(content, limit) -class ViewURLEncoded: +class ViewURLEncoded(View): name = "URL-encoded" prompt = ("urlencoded", "u") content_types = ["application/x-www-form-urlencoded"] def __call__(self, hdrs, content, limit): - lines = netlib.utils.urldecode(content) - if lines: - body = common.format_keyvals( - [(k + ":", v) for (k, v) in lines], - key = "header", - val = "text" - ) - return "URLEncoded form", body + d = netlib.utils.urldecode(content) + return "URLEncoded form", format_dict(d) -class ViewMultipart: +class ViewMultipart(View): name = "Multipart Form" prompt = ("multipart", "m") content_types = ["multipart/form-data"] + @staticmethod + def _format(v): + yield (("highlight", "Form data:\n")) + for message in format_dict({key:val for key,val in v}): + yield message + def __call__(self, hdrs, content, limit): v = netlib.utils.multipartdecode(hdrs, content) if v: - r = [ - urwid.Text(("highlight", "Form data:\n")), - ] - r.extend(common.format_keyvals( - v, - key = "header", - val = "text" - )) - return "Multipart form", r + return "Multipart form", self._format(v) if pyamf: @@ -263,6 +291,7 @@ if pyamf: data = input.readObject() self["data"] = data + def pyamf_class_loader(s): for i in pyamf.CLASS_LOADERS: if i != pyamf_class_loader: @@ -271,9 +300,11 @@ if pyamf: return v return DummyObject + pyamf.register_class_loader(pyamf_class_loader) - class ViewAMF: + + class ViewAMF(View): name = "AMF" prompt = ("amf", "f") content_types = ["application/x-amf"] @@ -300,31 +331,32 @@ if pyamf: else: return b - def __call__(self, hdrs, content, limit): - envelope = remoting.decode(content, strict=False) - if not envelope: - return None - - txt = [] + def _format(self, envelope, limit): for target, message in iter(envelope): if isinstance(message, pyamf.remoting.Request): - txt.append(urwid.Text([ + yield ( ("header", "Request: "), ("text", str(target)), - ])) + ) else: - txt.append(urwid.Text([ + yield ( ("header", "Response: "), ("text", "%s, code %s" % (target, message.status)), - ])) + ) s = json.dumps(self.unpack(message), indent=4) - txt.extend(_view_text(s[:limit], len(s), limit)) + for msg in format_text(s, limit): + yield msg + + def __call__(self, hdrs, content, limit): + envelope = remoting.decode(content, strict=False) + if not envelope: + return None - return "AMF v%s" % envelope.amfVersion, txt + return "AMF v%s" % envelope.amfVersion, self._format(envelope, limit) -class ViewJavaScript: +class ViewJavaScript(View): name = "JavaScript" prompt = ("javascript", "j") content_types = [ @@ -337,10 +369,11 @@ class ViewJavaScript: opts = jsbeautifier.default_options() opts.indent_size = 2 res = jsbeautifier.beautify(content[:limit], opts) - return "JavaScript", _view_text(res, len(res), limit) + cutoff = max(0, len(content) - limit) + return "JavaScript", format_text(res, limit - cutoff) -class ViewCSS: +class ViewCSS(View): name = "CSS" prompt = ("css", "c") content_types = [ @@ -354,10 +387,10 @@ class ViewCSS: else: beautified = content - return "CSS", _view_text(beautified, len(beautified), limit) + return "CSS", format_text(beautified, limit) -class ViewImage: +class ViewImage(View): name = "Image" prompt = ("image", "i") content_types = [ @@ -396,15 +429,11 @@ class ViewImage: clean.append( [netlib.utils.cleanBin(i[0]), netlib.utils.cleanBin(i[1])] ) - fmt = common.format_keyvals( - clean, - key = "header", - val = "text" - ) + fmt = format_dict({k:v for k,v in clean}) return "%s image" % img.format, fmt -class ViewProtobuf: +class ViewProtobuf(View): """Human friendly view of protocol buffers The view uses the protoc compiler to decode the binary """ @@ -443,11 +472,10 @@ class ViewProtobuf: def __call__(self, hdrs, content, limit): decoded = self.decode_protobuf(content) - txt = _view_text(decoded[:limit], len(decoded), limit) - return "Protobuf", txt + return "Protobuf", format_text(decoded, limit) -class ViewWBXML: +class ViewWBXML(View): name = "WBXML" prompt = ("wbxml", "w") content_types = [ @@ -460,11 +488,11 @@ class ViewWBXML: try: parser = ASCommandResponse(content) parsedContent = parser.xmlString - txt = _view_text(parsedContent, len(parsedContent), limit) - return "WBXML", txt + return "WBXML", format_text(parsedContent, limit) except: return None + views = [ ViewAuto(), ViewRaw(), @@ -492,7 +520,6 @@ for i in views: l = content_types_map.setdefault(ct, []) l.append(i) - view_prompts = [i.prompt for i in views] @@ -508,9 +535,13 @@ def get(name): return i -def get_content_view(viewmode, headers, content, limit, is_request): +def get_content_view(viewmode, headers, content, limit, is_request, log=None): """ - Returns a (msg, body) tuple. + Returns: + A (msg, body) tuple. + + Raises: + ContentViewException, if the content view threw an error. """ if not content: if is_request: @@ -529,9 +560,10 @@ def get_content_view(viewmode, headers, content, limit, is_request): ret = viewmode(headers, content, limit) # Third-party viewers can fail in unexpected ways... except Exception: - s = traceback.format_exc() - s = "Content viewer failed: \n" + s - signals.add_event(s, "error") + if log: + s = traceback.format_exc() + s = "Content viewer failed: \n" + s + log(s, "error") ret = None if not ret: ret = get("Raw")(headers, content, limit) diff --git a/test/test_console_contentview.py b/test/test_console_contentview.py index 6a93346a..d44a3cf4 100644 --- a/test/test_console_contentview.py +++ b/test/test_console_contentview.py @@ -9,7 +9,7 @@ import sys import netlib.utils from netlib import encoding -import libmproxy.console.contentview as cv +import libmproxy.contentview as cv from libmproxy import utils, flow import tutils diff --git a/test/test_console_import.py b/test/test_console_import.py new file mode 100644 index 00000000..c99faae8 --- /dev/null +++ b/test/test_console_import.py @@ -0,0 +1,5 @@ +import libmproxy.contentview as cv + + +def test_pass(): + assert True |