diff options
| -rw-r--r-- | .gitattributes | 2 | ||||
| -rw-r--r-- | docs/install.rst | 4 | ||||
| -rw-r--r-- | mitmproxy/contentviews/auto.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/image/__init__.py | 4 | ||||
| -rw-r--r-- | mitmproxy/contentviews/image/image_parser.py | 14 | ||||
| -rw-r--r-- | mitmproxy/contentviews/image/view.py | 49 | ||||
| -rw-r--r-- | setup.cfg | 31 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | test/conftest.py | 124 | ||||
| -rw-r--r-- | test/full_coverage_plugin.py | 119 | ||||
| -rw-r--r-- | test/mitmproxy/__init__.py | 1 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/image/test_image_parser.py | 6 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/image/test_view.py | 8 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_auto.py | 12 | ||||
| -rw-r--r-- | test/mitmproxy/proxy/protocol/test_http2.py | 2 | ||||
| -rw-r--r-- | tox.ini | 11 |
16 files changed, 215 insertions, 176 deletions
diff --git a/.gitattributes b/.gitattributes index dd08ee53..69d68b8e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,2 @@ -mitmproxy/tools/web/static/**/* -diff +mitmproxy/tools/web/static/**/* -diff linguist-vendored web/src/js/filt/filt.js -diff diff --git a/docs/install.rst b/docs/install.rst index b9524897..cf93cc58 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -85,7 +85,7 @@ libraries. This was tested on a fully patched installation of Ubuntu 16.04. .. code:: bash - sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev + sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev sudo pip3 install mitmproxy # or pip3 install --user mitmproxy On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install @@ -104,7 +104,7 @@ libraries. This was tested on a fully patched installation of Fedora 24. .. code:: bash - sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel + sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel sudo pip3 install mitmproxy # or pip3 install --user mitmproxy Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. diff --git a/mitmproxy/contentviews/auto.py b/mitmproxy/contentviews/auto.py index 7b3cbd78..d46a1bd3 100644 --- a/mitmproxy/contentviews/auto.py +++ b/mitmproxy/contentviews/auto.py @@ -18,6 +18,8 @@ class ViewAuto(base.View): return contentviews.content_types_map[ct][0](data, **metadata) elif strutils.is_xml(data): return contentviews.get("XML/HTML")(data, **metadata) + elif ct.startswith("image/"): + return contentviews.get("Image")(data, **metadata) if metadata.get("query"): return contentviews.get("Query")(data, **metadata) if data and strutils.is_mostly_bin(data): diff --git a/mitmproxy/contentviews/image/__init__.py b/mitmproxy/contentviews/image/__init__.py index 0d0f06e0..33356bd7 100644 --- a/mitmproxy/contentviews/image/__init__.py +++ b/mitmproxy/contentviews/image/__init__.py @@ -1 +1,3 @@ -from .view import ViewImage # noqa +from .view import ViewImage + +__all__ = ["ViewImage"] diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index 1ff3cff7..062fb38e 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -13,9 +13,9 @@ Metadata = typing.List[typing.Tuple[str, str]] def parse_png(data: bytes) -> Metadata: img = png.Png(KaitaiStream(io.BytesIO(data))) parts = [ - ('Format', 'Portable network graphics') + ('Format', 'Portable network graphics'), + ('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height)) ] - parts.append(('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height))) for chunk in img.chunks: if chunk.type == 'gAMA': parts.append(('gamma', str(chunk.body.gamma_int / 100000))) @@ -34,13 +34,13 @@ def parse_png(data: bytes) -> Metadata: def parse_gif(data: bytes) -> Metadata: img = gif.Gif(KaitaiStream(io.BytesIO(data))) + descriptor = img.logical_screen_descriptor parts = [ - ('Format', 'Compuserve GIF') + ('Format', 'Compuserve GIF'), + ('Version', "GIF{}".format(img.header.version.decode('ASCII'))), + ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)), + ('background', str(descriptor.bg_color_index)) ] - parts.append(('version', "GIF{0}".format(img.header.version.decode('ASCII')))) - descriptor = img.logical_screen_descriptor - parts.append(('Size', "{0} x {1} px".format(descriptor.screen_width, descriptor.screen_height))) - parts.append(('background', str(descriptor.bg_color_index))) ext_blocks = [] for block in img.blocks: if block.block_type.name == 'extension': diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index 8fdb26e9..95ee1e43 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -1,55 +1,38 @@ -import io import imghdr -from PIL import Image - +from mitmproxy.contentviews import base from mitmproxy.types import multidict from . import image_parser -from mitmproxy.contentviews import base - class ViewImage(base.View): name = "Image" prompt = ("image", "i") + + # there is also a fallback in the auto view for image/*. content_types = [ "image/png", "image/jpeg", "image/gif", "image/vnd.microsoft.icon", "image/x-icon", + "image/webp", ] def __call__(self, data, **metadata): image_type = imghdr.what('', h=data) if image_type == 'png': - f = "PNG" - parts = image_parser.parse_png(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_png(data) elif image_type == 'gif': - f = "GIF" - parts = image_parser.parse_gif(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_gif(data) elif image_type == 'jpeg': - f = "JPEG" - parts = image_parser.parse_jpeg(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt - try: - img = Image.open(io.BytesIO(data)) - except IOError: - return None - parts = [ - ("Format", str(img.format_description)), - ("Size", "%s x %s px" % img.size), - ("Mode", str(img.mode)), - ] - for i in sorted(img.info.keys()): - if i != "exif": - parts.append( - (str(i), str(img.info[i])) - ) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % img.format, fmt + image_metadata = image_parser.parse_jpeg(data) + else: + image_metadata = [ + ("Image Format", image_type or "unknown") + ] + if image_type: + view_name = "{} Image".format(image_type.upper()) + else: + view_name = "Unknown Image" + return view_name, base.format_dict(multidict.MultiDict(image_metadata)) @@ -18,3 +18,34 @@ show_missing = True exclude_lines = pragma: no cover raise NotImplementedError() + +[tool:full_coverage] +exclude = + mitmproxy/contentviews/__init__.py + mitmproxy/contentviews/protobuf.py + mitmproxy/contentviews/wbxml.py + mitmproxy/contentviews/xml_html.py + mitmproxy/net/tcp.py + mitmproxy/net/http/cookies.py + mitmproxy/net/http/encoding.py + mitmproxy/net/http/message.py + mitmproxy/net/http/url.py + mitmproxy/proxy/protocol/ + mitmproxy/proxy/config.py + mitmproxy/proxy/root_context.py + mitmproxy/proxy/server.py + mitmproxy/tools/ + mitmproxy/certs.py + mitmproxy/connections.py + mitmproxy/controller.py + mitmproxy/export.py + mitmproxy/flow.py + mitmproxy/flowfilter.py + mitmproxy/http.py + mitmproxy/io_compat.py + mitmproxy/master.py + mitmproxy/optmanager.py + pathod/pathoc.py + pathod/pathod.py + pathod/test.py + pathod/protocols/http2.py @@ -71,7 +71,6 @@ setup( "hyperframe>=4.0.1, <5", "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.6, <0.7", - "Pillow>=3.2, <4.1", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", "pyOpenSSL>=16.0, <17.0", @@ -118,6 +117,7 @@ setup( 'examples': [ "beautifulsoup4>=4.4.1, <4.6", "pytz>=2015.07.0, <=2016.10", + "Pillow>=3.2, <4.1", ] } ) diff --git a/test/conftest.py b/test/conftest.py index 83823a19..b4e1da93 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,6 +6,7 @@ from contextlib import contextmanager import mitmproxy.net.tcp +pytest_plugins = ('test.full_coverage_plugin',) requires_alpn = pytest.mark.skipif( not mitmproxy.net.tcp.HAS_ALPN, @@ -27,10 +28,17 @@ skip_appveyor = pytest.mark.skipif( ) -original_pytest_raises = pytest.raises +@pytest.fixture() +def disable_alpn(monkeypatch): + monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False) + monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False) +################################################################################ # TODO: remove this wrapper when pytest 3.1.0 is released +original_pytest_raises = pytest.raises + + @contextmanager @functools.wraps(original_pytest_raises) def raises(exc, *args, **kwargs): @@ -41,116 +49,4 @@ def raises(exc, *args, **kwargs): pytest.raises = raises - - -@pytest.fixture() -def disable_alpn(monkeypatch): - monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False) - monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False) - - -enable_coverage = False -coverage_values = [] -coverage_passed = False - - -def pytest_addoption(parser): - parser.addoption('--full-cov', - action='append', - dest='full_cov', - default=[], - help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none") - - parser.addoption('--no-full-cov', - action='append', - dest='no_full_cov', - default=[], - help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none") - - -def pytest_configure(config): - global enable_coverage - enable_coverage = ( - len(config.getoption('file_or_dir')) == 0 and - len(config.getoption('full_cov')) > 0 and - config.pluginmanager.getplugin("_cov") is not None and - config.pluginmanager.getplugin("_cov").cov_controller is not None and - config.pluginmanager.getplugin("_cov").cov_controller.cov is not None - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtestloop(session): - global enable_coverage - global coverage_values - global coverage_passed - - if not enable_coverage: - yield - return - - cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov - - if os.name == 'nt': - cov.exclude('pragma: windows no cover') - - yield - - coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) - - prefix = os.getcwd() - excluded_files = [os.path.normpath(f) for f in pytest.config.option.no_full_cov] - measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()] - measured_files = [f for f in measured_files if not any(f.startswith(excluded_f) for excluded_f in excluded_files)] - - for name in coverage_values.keys(): - files = [f for f in measured_files if f.startswith(os.path.normpath(name))] - try: - with open(os.devnull, 'w') as null: - overall = cov.report(files, ignore_errors=True, file=null) - singles = [(s, cov.report(s, ignore_errors=True, file=null)) for s in files] - coverage_values[name] = (overall, singles) - except: - pass - - if any(v < 100 for v, _ in coverage_values.values()): - # make sure we get the EXIT_TESTSFAILED exit code - session.testsfailed += 1 - else: - coverage_passed = True - - -def pytest_terminal_summary(terminalreporter, exitstatus): - global enable_coverage - global coverage_values - global coverage_passed - - if not enable_coverage: - return - - terminalreporter.write('\n') - if not coverage_passed: - markup = {'red': True, 'bold': True} - msg = "FAIL: Full test coverage not reached!\n" - terminalreporter.write(msg, **markup) - - for name in sorted(coverage_values.keys()): - msg = 'Coverage for {}: {:.2f}%\n'.format(name, coverage_values[name][0]) - if coverage_values[name][0] < 100: - markup = {'red': True, 'bold': True} - for s, v in sorted(coverage_values[name][1]): - if v < 100: - msg += ' {}: {:.2f}%\n'.format(s, v) - else: - markup = {'green': True} - terminalreporter.write(msg, **markup) - else: - markup = {'green': True} - msg = 'SUCCESS: Full test coverage reached in modules and files:\n' - msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) - terminalreporter.write(msg, **markup) - - msg = '\nExcluded files:\n' - for s in sorted(pytest.config.option.no_full_cov): - msg += " {}\n".format(s) - terminalreporter.write(msg) +################################################################################ diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py new file mode 100644 index 00000000..e9951af9 --- /dev/null +++ b/test/full_coverage_plugin.py @@ -0,0 +1,119 @@ +import os +import configparser +import pytest + + +enable_coverage = False +coverage_values = [] +coverage_passed = True +no_full_cov = [] + + +def pytest_addoption(parser): + parser.addoption('--full-cov', + action='append', + dest='full_cov', + default=[], + help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none") + + parser.addoption('--no-full-cov', + action='append', + dest='no_full_cov', + default=[], + help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none") + + +def pytest_configure(config): + global enable_coverage + global no_full_cov + + enable_coverage = ( + len(config.getoption('file_or_dir')) == 0 and + len(config.getoption('full_cov')) > 0 and + config.pluginmanager.getplugin("_cov") is not None and + config.pluginmanager.getplugin("_cov").cov_controller is not None and + config.pluginmanager.getplugin("_cov").cov_controller.cov is not None + ) + + c = configparser.ConfigParser() + c.read('setup.cfg') + fs = c['tool:full_coverage']['exclude'].split('\n') + no_full_cov = config.option.no_full_cov + [f.strip() for f in fs] + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtestloop(session): + global enable_coverage + global coverage_values + global coverage_passed + global no_full_cov + + if not enable_coverage: + yield + return + + cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov + + if os.name == 'nt': + cov.exclude('pragma: windows no cover') + + yield + + coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) + + prefix = os.getcwd() + + excluded_files = [os.path.normpath(f) for f in no_full_cov] + measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()] + measured_files = [f for f in measured_files if not any(f.startswith(excluded_f) for excluded_f in excluded_files)] + + for name in coverage_values.keys(): + files = [f for f in measured_files if f.startswith(os.path.normpath(name))] + try: + with open(os.devnull, 'w') as null: + overall = cov.report(files, ignore_errors=True, file=null) + singles = [(s, cov.report(s, ignore_errors=True, file=null)) for s in files] + coverage_values[name] = (overall, singles) + except: + pass + + if any(v < 100 for v, _ in coverage_values.values()): + # make sure we get the EXIT_TESTSFAILED exit code + session.testsfailed += 1 + coverage_passed = False + + +def pytest_terminal_summary(terminalreporter, exitstatus): + global enable_coverage + global coverage_values + global coverage_passed + global no_full_cov + + if not enable_coverage: + return + + terminalreporter.write('\n') + if not coverage_passed: + markup = {'red': True, 'bold': True} + msg = "FAIL: Full test coverage not reached!\n" + terminalreporter.write(msg, **markup) + + for name in sorted(coverage_values.keys()): + msg = 'Coverage for {}: {:.2f}%\n'.format(name, coverage_values[name][0]) + if coverage_values[name][0] < 100: + markup = {'red': True, 'bold': True} + for s, v in sorted(coverage_values[name][1]): + if v < 100: + msg += ' {}: {:.2f}%\n'.format(s, v) + else: + markup = {'green': True} + terminalreporter.write(msg, **markup) + else: + msg = 'SUCCESS: Full test coverage reached in modules and files:\n' + msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) + terminalreporter.write(msg, green=True) + + msg = '\nExcluded files:\n' + for s in sorted(no_full_cov): + msg += " {}\n".format(s) + terminalreporter.write(msg) diff --git a/test/mitmproxy/__init__.py b/test/mitmproxy/__init__.py index 28dc133f..6f114e18 100644 --- a/test/mitmproxy/__init__.py +++ b/test/mitmproxy/__init__.py @@ -3,5 +3,4 @@ import logging logging.getLogger("hyper").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("passlib").setLevel(logging.WARNING) -logging.getLogger("PIL").setLevel(logging.WARNING) logging.getLogger("tornado").setLevel(logging.WARNING) diff --git a/test/mitmproxy/contentviews/image/test_image_parser.py b/test/mitmproxy/contentviews/image/test_image_parser.py index 3c8bfdf7..3cb44ca6 100644 --- a/test/mitmproxy/contentviews/image/test_image_parser.py +++ b/test/mitmproxy/contentviews/image/test_image_parser.py @@ -80,7 +80,7 @@ def test_parse_png(filename, metadata): # check comment "mitmproxy/data/image_parser/hopper.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '128 x 128 px'), ('background', '0'), ('comment', "b'File written by Adobe Photoshop\\xa8 4.0'") @@ -88,7 +88,7 @@ def test_parse_png(filename, metadata): # check background "mitmproxy/data/image_parser/chi.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '320 x 240 px'), ('background', '248'), ('comment', "b'Created with GIMP'") @@ -96,7 +96,7 @@ def test_parse_png(filename, metadata): # check working with color table "mitmproxy/data/image_parser/iss634.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '245 x 245 px'), ('background', '0') ], diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py index ee2f9eaa..34f655a1 100644 --- a/test/mitmproxy/contentviews/image/test_view.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -9,9 +9,11 @@ def test_view_image(): "mitmproxy/data/image.png", "mitmproxy/data/image.gif", "mitmproxy/data/all.jpeg", - "mitmproxy/data/image.ico" + # https://bugs.python.org/issue21574 + # "mitmproxy/data/image.ico", ]: with open(tutils.test_data.path(img), "rb") as f: - assert v(f.read()) + viewname, lines = v(f.read()) + assert img.split(".")[-1].upper() in viewname - assert not v(b"flibble") + assert v(b"flibble") == ('Unknown Image', [[('header', 'Image Format: '), ('text', 'unknown')]]) diff --git a/test/mitmproxy/contentviews/test_auto.py b/test/mitmproxy/contentviews/test_auto.py index a077affa..2ff43139 100644 --- a/test/mitmproxy/contentviews/test_auto.py +++ b/test/mitmproxy/contentviews/test_auto.py @@ -30,6 +30,18 @@ def test_view_auto(): ) assert f[0].startswith("XML") + f = v( + b"<svg></svg>", + headers=http.Headers(content_type="image/svg+xml") + ) + assert f[0].startswith("XML") + + f = v( + b"verybinary", + headers=http.Headers(content_type="image/new-magic-image-format") + ) + assert f[0] == "Unknown Image" + f = v(b"\xFF" * 30) assert f[0] == "Hex" diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index f5d9259d..cede0b80 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -23,8 +23,6 @@ logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) logging.getLogger("passlib.utils.compat").setLevel(logging.WARNING) logging.getLogger("passlib.registry").setLevel(logging.WARNING) -logging.getLogger("PIL.Image").setLevel(logging.WARNING) -logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) # inspect the log: @@ -11,14 +11,9 @@ passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* O setenv = HOME = {envtmpdir} commands = mitmdump --version - pytest --timeout 60 --cov-report='' --cov=mitmproxy --cov=pathod \ - --full-cov=mitmproxy/ \ - --no-full-cov=mitmproxy/contentviews/__init__.py --no-full-cov=mitmproxy/contentviews/protobuf.py --no-full-cov=mitmproxy/contentviews/wbxml.py --no-full-cov=mitmproxy/contentviews/xml_html.py \ - --no-full-cov=mitmproxy/net/tcp.py --no-full-cov=mitmproxy/net/http/cookies.py --no-full-cov=mitmproxy/net/http/encoding.py --no-full-cov=mitmproxy/net/http/message.py --no-full-cov=mitmproxy/net/http/url.py \ - --no-full-cov=mitmproxy/proxy/protocol/ --no-full-cov=mitmproxy/proxy/config.py --no-full-cov=mitmproxy/proxy/root_context.py --no-full-cov=mitmproxy/proxy/server.py \ - --no-full-cov=mitmproxy/tools/ \ - --no-full-cov=mitmproxy/certs.py --no-full-cov=mitmproxy/connections.py --no-full-cov=mitmproxy/controller.py --no-full-cov=mitmproxy/export.py --no-full-cov=mitmproxy/flow.py --no-full-cov=mitmproxy/flowfilter.py --no-full-cov=mitmproxy/http.py --no-full-cov=mitmproxy/io_compat.py --no-full-cov=mitmproxy/master.py --no-full-cov=mitmproxy/optmanager.py \ - --full-cov=pathod/ --no-full-cov=pathod/pathoc.py --no-full-cov=pathod/pathod.py --no-full-cov=pathod/test.py --no-full-cov=pathod/protocols/http2.py \ + pytest --timeout 60 --cov-report='' \ + --cov=mitmproxy --cov=pathod \ + --full-cov=mitmproxy/ --full-cov=pathod/ \ {posargs} {env:CI_COMMANDS:python -c ""} |