Merge pull request #1959 from Kriechi/coverage-fail

add test coverage protection

7 files changed, 235 insertions, 6 deletions

diff --git a/test/conftest.py b/test/conftest.py
index 3d129ecf..4d779b01 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,5 +1,7 @@
+import os
 import pytest
 import OpenSSL
+
 import mitmproxy.net.tcp
 
 
@@ -12,3 +14,104 @@ requires_alpn = pytest.mark.skipif(
 def disable_alpn(monkeypatch):
     monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False)
     monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False)
+
+
+enable_coverage = False
+coverage_values = []
+coverage_passed = False
+
+
+def pytest_addoption(parser):
+    parser.addoption('--full-cov',
+                     action='append',
+                     dest='full_cov',
+                     default=[],
+                     help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none")
+
+    parser.addoption('--no-full-cov',
+                     action='append',
+                     dest='no_full_cov',
+                     default=[],
+                     help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none")
+
+
+def pytest_configure(config):
+    global enable_coverage
+    enable_coverage = (
+        len(config.getoption('file_or_dir')) == 0 and
+        len(config.getoption('full_cov')) > 0 and
+        config.pluginmanager.getplugin("_cov") is not None and
+        config.pluginmanager.getplugin("_cov").cov_controller is not None and
+        config.pluginmanager.getplugin("_cov").cov_controller.cov is not None
+    )
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtestloop(session):
+    global enable_coverage
+    global coverage_values
+    global coverage_passed
+
+    if not enable_coverage:
+        yield
+        return
+
+    cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov
+
+    if os.name == 'nt':
+        cov.exclude('pragma: windows no cover')
+
+    yield
+
+    coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov])
+
+    prefix = os.getcwd()
+    excluded_files = [os.path.normpath(f) for f in pytest.config.option.no_full_cov]
+    measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()]
+    measured_files = [f for f in measured_files if f not in excluded_files]
+
+    for name in pytest.config.option.full_cov:
+        files = [f for f in measured_files if f.startswith(os.path.normpath(name))]
+        try:
+            with open(os.devnull, 'w') as null:
+                coverage_values[name] = cov.report(files, ignore_errors=True, file=null)
+        except:
+            pass
+
+    if any(v < 100 for v in coverage_values.values()):
+        # make sure we get the EXIT_TESTSFAILED exit code
+        session.testsfailed += 1
+    else:
+        coverage_passed = True
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus):
+    global enable_coverage
+    global coverage_values
+    global coverage_passed
+
+    if not enable_coverage:
+        return
+
+    terminalreporter.write('\n')
+    if not coverage_passed:
+        markup = {'red': True, 'bold': True}
+        msg = "FAIL: Full test coverage not reached!\n"
+        terminalreporter.write(msg, **markup)
+
+        for name, value in coverage_values.items():
+            if value < 100:
+                markup = {'red': True, 'bold': True}
+            else:
+                markup = {'green': True}
+            msg = 'Coverage for {}: {:.2f}%\n'.format(name, value)
+            terminalreporter.write(msg, **markup)
+    else:
+        markup = {'green': True}
+        msg = 'SUCCESS: Full test coverage reached in modules and files:\n'
+        msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov))
+        terminalreporter.write(msg, **markup)
+
+    msg = 'Excluded files:\n'
+    msg += '{}\n'.format('\n'.join(pytest.config.option.no_full_cov))
+    terminalreporter.write(msg)
diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py
index da4ca007..6a7f2ef1 100644
--- a/test/mitmproxy/addons/test_view.py
+++ b/test/mitmproxy/addons/test_view.py
@@ -76,7 +76,7 @@ def test_simple():
     assert v.get_by_id(f.id)
     assert not v.get_by_id("nonexistent")
 
-    # These all just call udpate
+    # These all just call update
     v.error(f)
     v.response(f)
     v.intercept(f)
diff --git a/test/mitmproxy/net/http/http2/test_framereader.py b/test/mitmproxy/net/http/http2/test_framereader.py
index 41b73189..485ba69f 100644
--- a/test/mitmproxy/net/http/http2/test_framereader.py
+++ b/test/mitmproxy/net/http/http2/test_framereader.py
@@ -1 +1,41 @@
-# foobar
+import pytest
+import codecs
+from io import BytesIO
+import hyperframe.frame
+
+from mitmproxy import exceptions
+from mitmproxy.net.http.http2 import read_raw_frame, parse_frame
+
+
+def test_read_raw_frame():
+    raw = codecs.decode('000006000101234567666f6f626172', 'hex_codec')
+    bio = BytesIO(raw)
+    bio.safe_read = bio.read
+
+    header, body = read_raw_frame(bio)
+    assert header
+    assert body
+
+
+def test_read_raw_frame_failed():
+    raw = codecs.decode('485454000000000000', 'hex_codec')
+    bio = BytesIO(raw)
+    bio.safe_read = bio.read
+
+    with pytest.raises(exceptions.HttpException):
+        read_raw_frame(bio)
+
+
+def test_parse_frame():
+    f = parse_frame(
+        codecs.decode('000006000101234567', 'hex_codec'),
+        codecs.decode('666f6f626172', 'hex_codec')
+    )
+    assert isinstance(f, hyperframe.frame.Frame)
+
+
+def test_parse_frame_combined():
+    f = parse_frame(
+        codecs.decode('000006000101234567666f6f626172', 'hex_codec'),
+    )
+    assert isinstance(f, hyperframe.frame.Frame)
diff --git a/test/mitmproxy/net/http/http2/test_utils.py b/test/mitmproxy/net/http/http2/test_utils.py
new file mode 100644
index 00000000..41d49b6f
--- /dev/null
+++ b/test/mitmproxy/net/http/http2/test_utils.py
@@ -0,0 +1,70 @@
+import pytest
+
+from mitmproxy.net.http.http2 import parse_headers
+
+
+class TestHttp2ParseHeaders:
+
+    def test_relative(self):
+        h = dict([
+            (':authority', "127.0.0.1:1234"),
+            (':method', 'GET'),
+            (':scheme', 'https'),
+            (':path', '/'),
+        ])
+        first_line_format, method, scheme, host, port, path = parse_headers(h)
+        assert first_line_format == 'relative'
+        assert method == b'GET'
+        assert scheme == b'https'
+        assert host == b'127.0.0.1'
+        assert port == 1234
+        assert path == b'/'
+
+    def test_absolute(self):
+        h = dict([
+            (':authority', "127.0.0.1:1234"),
+            (':method', 'GET'),
+            (':scheme', 'https'),
+            (':path', 'https://127.0.0.1:4321'),
+        ])
+        first_line_format, method, scheme, host, port, path = parse_headers(h)
+        assert first_line_format == 'absolute'
+        assert method == b'GET'
+        assert scheme == b'https'
+        assert host == b'127.0.0.1'
+        assert port == 1234
+        assert path == b'https://127.0.0.1:4321'
+
+    @pytest.mark.parametrize("scheme, expected_port", [
+        ('http', 80),
+        ('https', 443),
+    ])
+    def test_without_port(self, scheme, expected_port):
+        h = dict([
+            (':authority', "127.0.0.1"),
+            (':method', 'GET'),
+            (':scheme', scheme),
+            (':path', '/'),
+        ])
+        _, _, _, _, port, _ = parse_headers(h)
+        assert port == expected_port
+
+    def test_without_authority(self):
+        h = dict([
+            (':method', 'GET'),
+            (':scheme', 'https'),
+            (':path', '/'),
+        ])
+        _, _, _, host, _, _ = parse_headers(h)
+        assert host == b'localhost'
+
+    def test_connect(self):
+        h = dict([
+            (':authority', "127.0.0.1"),
+            (':method', 'CONNECT'),
+            (':scheme', 'https'),
+            (':path', '/'),
+        ])
+
+        with pytest.raises(NotImplementedError):
+            parse_headers(h)
diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py
index bb760f92..90cdc3d8 100644
--- a/test/mitmproxy/script/test_concurrent.py
+++ b/test/mitmproxy/script/test_concurrent.py
@@ -8,7 +8,6 @@ from mitmproxy.addons import script
 import time
 
 from test.mitmproxy import mastertest
-from test.mitmproxy import tutils as ttutils
 
 
 class Thing:
@@ -18,7 +17,6 @@ class Thing:
 
 
 class TestConcurrent(mastertest.MasterTest):
-    @ttutils.skip_appveyor
     def test_concurrent(self):
         with taddons.context() as tctx:
             sc = script.Script(
diff --git a/test/mitmproxy/utils/test_debug.py b/test/mitmproxy/utils/test_debug.py
index 18f5cdbc..22f8dc66 100644
--- a/test/mitmproxy/utils/test_debug.py
+++ b/test/mitmproxy/utils/test_debug.py
@@ -1,4 +1,6 @@
 import io
+import subprocess
+from unittest import mock
 
 from mitmproxy.utils import debug
 
@@ -6,6 +8,10 @@ from mitmproxy.utils import debug
 def test_dump_system_info():
     assert debug.dump_system_info()
 
+    with mock.patch('subprocess.check_output') as m:
+        m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long')
+        assert 'release version' in debug.dump_system_info()
+
 
 def test_dump_info():
     cs = io.StringIO()
diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py
index a21c65c9..67981be4 100644
--- a/test/mitmproxy/utils/test_typecheck.py
+++ b/test/mitmproxy/utils/test_typecheck.py
@@ -38,8 +38,15 @@ def test_check_union():
     with pytest.raises(TypeError):
         typecheck.check_type("foo", [], typing.Union[int, str])
 
+    # Python 3.5 only defines __union_params__
+    m = mock.Mock()
+    m.__str__ = lambda self: "typing.Union"
+    m.__union_params__ = (int,)
+    typecheck.check_type("foo", 42, m)
+
 
 def test_check_tuple():
+    typecheck.check_type("foo", (42, "42"), typing.Tuple[int, str])
     with pytest.raises(TypeError):
         typecheck.check_type("foo", None, typing.Tuple[int, str])
     with pytest.raises(TypeError):
@@ -48,7 +55,12 @@ def test_check_tuple():
         typecheck.check_type("foo", (42, 42), typing.Tuple[int, str])
     with pytest.raises(TypeError):
         typecheck.check_type("foo", ("42", 42), typing.Tuple[int, str])
-    typecheck.check_type("foo", (42, "42"), typing.Tuple[int, str])
+
+    # Python 3.5 only defines __tuple_params__
+    m = mock.Mock()
+    m.__str__ = lambda self: "typing.Tuple"
+    m.__tuple_params__ = (int, str)
+    typecheck.check_type("foo", (42, "42"), m)
 
 
 def test_check_sequence():
@@ -62,7 +74,7 @@ def test_check_sequence():
     with pytest.raises(TypeError):
         typecheck.check_type("foo", "foo", typing.Sequence[str])
 
-    # Python 3.5.0 only defines __parameters__
+    # Python 3.5 only defines __parameters__
     m = mock.Mock()
     m.__str__ = lambda self: "typing.Sequence"
     m.__parameters__ = (int,)