diff options
27 files changed, 364 insertions, 352 deletions
diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index 696542f6..a6844241 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -8,26 +8,11 @@ import configargparse from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import platform +from mitmproxy import options from netlib import human from netlib import tcp from netlib import version -APP_HOST = "mitm.it" -APP_PORT = 80 -CA_DIR = "~/.mitmproxy" - -# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. -# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old -DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \ - "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \ - "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \ - "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \ - "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \ - "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \ - "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \ - "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \ - "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" - class ParseException(Exception): pass @@ -222,6 +207,9 @@ def get_common_options(args): "the client certificate chain." ) + if args.quiet: + args.verbose = 0 + return dict( app=args.app, app_host=args.app_host, @@ -307,8 +295,8 @@ def basic_options(parser): ) parser.add_argument( "--cadir", - action="store", type=str, dest="cadir", default=CA_DIR, - help="Location of the default mitmproxy CA files. (%s)" % CA_DIR + action="store", type=str, dest="cadir", default=options.CA_DIR, + help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR ) parser.add_argument( "--host", @@ -459,7 +447,7 @@ def proxy_options(parser): ) group.add_argument( "-p", "--port", - action="store", type=int, dest="port", default=8080, + action="store", type=int, dest="port", default=options.LISTEN_PORT, help="Proxy service port." ) group.add_argument( @@ -506,7 +494,7 @@ def proxy_ssl_options(parser): 'as the first entry. Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, + type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS, help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( @@ -572,18 +560,18 @@ def onboarding_app(parser): ) group.add_argument( "--app-host", - action="store", dest="app_host", default=APP_HOST, metavar="host", + action="store", dest="app_host", default=options.APP_HOST, metavar="host", help=""" Domain to serve the onboarding app from. For transparent mode, use an IP when a DNS entry for the app domain is not present. Default: %s - """ % APP_HOST + """ % options.APP_HOST ) group.add_argument( "--app-port", action="store", dest="app_port", - default=APP_PORT, + default=options.APP_PORT, type=int, metavar="80", help="Port to serve the onboarding app from." @@ -761,8 +749,8 @@ def mitmproxy(): usage="%(prog)s [options]", args_for_setting_config_path=["--conf"], default_config_files=[ - os.path.join(CA_DIR, "common.conf"), - os.path.join(CA_DIR, "mitmproxy.conf") + os.path.join(options.CA_DIR, "common.conf"), + os.path.join(options.CA_DIR, "mitmproxy.conf") ], add_config_file_help=True, add_env_var_help=True @@ -816,8 +804,8 @@ def mitmdump(): usage="%(prog)s [options] [filter]", args_for_setting_config_path=["--conf"], default_config_files=[ - os.path.join(CA_DIR, "common.conf"), - os.path.join(CA_DIR, "mitmdump.conf") + os.path.join(options.CA_DIR, "common.conf"), + os.path.join(options.CA_DIR, "mitmdump.conf") ], add_config_file_help=True, add_env_var_help=True @@ -846,8 +834,8 @@ def mitmweb(): usage="%(prog)s [options]", args_for_setting_config_path=["--conf"], default_config_files=[ - os.path.join(CA_DIR, "common.conf"), - os.path.join(CA_DIR, "mitmweb.conf") + os.path.join(options.CA_DIR, "common.conf"), + os.path.join(options.CA_DIR, "mitmweb.conf") ], add_config_file_help=True, add_env_var_help=True diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 86e889cc..59d07456 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -23,6 +23,7 @@ from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import script from mitmproxy import utils +import mitmproxy.options from mitmproxy.console import flowlist from mitmproxy.console import flowview from mitmproxy.console import grideditor @@ -178,7 +179,7 @@ class ConsoleState(flow.State): self.add_flow_setting(flow, "marked", marked) -class Options(flow.options.Options): +class Options(mitmproxy.options.Options): def __init__( self, eventlog=False, # type: bool diff --git a/mitmproxy/contrib/tnetstring.py b/mitmproxy/contrib/tnetstring.py index d99a83f9..86236caa 100644 --- a/mitmproxy/contrib/tnetstring.py +++ b/mitmproxy/contrib/tnetstring.py @@ -188,9 +188,9 @@ def load(file_handle): def parse(data_type, data): + # type: (int, bytes) -> TSerializable if six.PY2: data_type = ord(data_type) - # type: (int, bytes) -> TSerializable if data_type == ord(b','): return data if data_type == ord(b';'): diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 78dd2578..4f34ab95 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -12,6 +12,7 @@ from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import builtins from mitmproxy import utils +from mitmproxy import options from mitmproxy.builtins import dumper from netlib import tcp @@ -20,7 +21,7 @@ class DumpError(Exception): pass -class Options(flow.options.Options): +class Options(options.Options): def __init__( self, keepserving=False, # type: bool diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index b2ab74c6..8a64180e 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -7,7 +7,6 @@ from mitmproxy.flow.modules import ( AppRegistry, StreamLargeBodies, ClientPlaybackState, ServerPlaybackState ) from mitmproxy.flow.state import State, FlowView -from mitmproxy.flow import options # TODO: We may want to remove the imports from .modules and just expose "modules" @@ -16,5 +15,5 @@ __all__ = [ "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", "FlowMaster", "AppRegistry", "StreamLargeBodies", "ClientPlaybackState", - "ServerPlaybackState", "State", "FlowView", "options", + "ServerPlaybackState", "State", "FlowView", ] diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/flow/io_compat.py index bcfbd375..8cd883c3 100644 --- a/mitmproxy/flow/io_compat.py +++ b/mitmproxy/flow/io_compat.py @@ -4,6 +4,7 @@ This module handles the import of mitmproxy flows generated by old versions. from __future__ import absolute_import, print_function, division import six +from typing import Any # noqa from netlib import version, strutils diff --git a/mitmproxy/flow/options.py b/mitmproxy/flow/options.py deleted file mode 100644 index 726952e2..00000000 --- a/mitmproxy/flow/options.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import absolute_import, print_function, division -from mitmproxy import options -from typing import Tuple, Optional, Sequence # noqa -from mitmproxy import cmdline - -APP_HOST = "mitm.it" -APP_PORT = 80 - - -class Options(options.Options): - def __init__( - self, - # TODO: rename to onboarding_app_* - app=True, # type: bool - app_host=APP_HOST, # type: str - app_port=APP_PORT, # type: int - anticache=False, # type: bool - anticomp=False, # type: bool - client_replay=None, # type: Optional[str] - kill=False, # type: bool - no_server=False, # type: bool - nopop=False, # type: bool - refresh_server_playback=False, # type: bool - rfile=None, # type: Optional[str] - scripts=(), # type: Sequence[str] - showhost=False, # type: bool - replacements=(), # type: Sequence[Tuple[str, str, str]] - rheaders=(), # type: Sequence[str] - setheaders=(), # type: Sequence[Tuple[str, str, str]] - server_replay=None, # type: Optional[str] - stickycookie=None, # type: Optional[str] - stickyauth=None, # type: Optional[str] - stream_large_bodies=None, # type: Optional[str] - verbosity=2, # type: int - outfile=None, # type: Tuple[str, str] - replay_ignore_content=False, # type: bool - replay_ignore_params=(), # type: Sequence[str] - replay_ignore_payload_params=(), # type: Sequence[str] - replay_ignore_host=False, # type: bool - - # Proxy options - auth_nonanonymous=False, # type: bool - auth_singleuser=None, # type: Optional[str] - auth_htpasswd=None, # type: Optional[str] - add_upstream_certs_to_client_chain=False, # type: bool - body_size_limit=None, # type: Optional[int] - cadir = cmdline.CA_DIR, # type: str - certs = (), # type: Sequence[Tuple[str, str]] - ciphers_client = cmdline.DEFAULT_CLIENT_CIPHERS, # type: str - ciphers_server = None, # type: Optional[str] - clientcerts = None, # type: Optional[str] - http2 = True, # type: bool - ignore_hosts = (), # type: Sequence[str] - listen_host = "", # type: str - listen_port = 8080, # type: int - mode = "regular", # type: str - no_upstream_cert = False, # type: bool - rawtcp = False, # type: bool - upstream_server = "", # type: str - upstream_auth = "", # type: str - ssl_version_client="secure", # type: str - ssl_version_server="secure", # type: str - ssl_verify_upstream_cert=False, # type: bool - ssl_verify_upstream_trusted_cadir=None, # type: str - ssl_verify_upstream_trusted_ca=None, # type: str - tcp_hosts = (), # type: Sequence[str] - ): - # We could replace all assignments with clever metaprogramming, - # but type hints are a much more valueable asset. - - self.app = app - self.app_host = app_host - self.app_port = app_port - self.anticache = anticache - self.anticomp = anticomp - self.client_replay = client_replay - self.kill = kill - self.no_server = no_server - self.nopop = nopop - self.refresh_server_playback = refresh_server_playback - self.rfile = rfile - self.scripts = scripts - self.showhost = showhost - self.replacements = replacements - self.rheaders = rheaders - self.setheaders = setheaders - self.server_replay = server_replay - self.stickycookie = stickycookie - self.stickyauth = stickyauth - self.stream_large_bodies = stream_large_bodies - self.verbosity = verbosity - self.outfile = outfile - self.replay_ignore_content = replay_ignore_content - self.replay_ignore_params = replay_ignore_params - self.replay_ignore_payload_params = replay_ignore_payload_params - self.replay_ignore_host = replay_ignore_host - - # Proxy options - self.auth_nonanonymous = auth_nonanonymous - self.auth_singleuser = auth_singleuser - self.auth_htpasswd = auth_htpasswd - self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain - self.body_size_limit = body_size_limit - self.cadir = cadir - self.certs = certs - self.ciphers_client = ciphers_client - self.ciphers_server = ciphers_server - self.clientcerts = clientcerts - self.http2 = http2 - self.ignore_hosts = ignore_hosts - self.listen_host = listen_host - self.listen_port = listen_port - self.mode = mode - self.no_upstream_cert = no_upstream_cert - self.rawtcp = rawtcp - self.upstream_server = upstream_server - self.upstream_auth = upstream_auth - self.ssl_version_client = ssl_version_client - self.ssl_version_server = ssl_version_server - self.ssl_verify_upstream_cert = ssl_verify_upstream_cert - self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir - self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca - self.tcp_hosts = tcp_hosts - super(Options, self).__init__() diff --git a/mitmproxy/main.py b/mitmproxy/main.py index 2d7299e4..6d44108e 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -30,25 +30,20 @@ def assert_utf8_env(): sys.exit(1) -def get_server(dummy_server, options): - if dummy_server: - return server.DummyServer(options) - else: - try: - return server.ProxyServer(options) - except exceptions.ServerException as v: - print(str(v), file=sys.stderr) - sys.exit(1) - - def process_options(parser, options, args): if args.sysinfo: print(debug.sysinfo()) sys.exit(0) - if args.quiet: - args.verbose = 0 debug.register_info_dumpers() - return config.ProxyConfig(options) + pconf = config.ProxyConfig(options) + if options.no_server: + return server.DummyServer(pconf) + else: + try: + return server.ProxyServer(pconf) + except exceptions.ServerException as v: + print(str(v), file=sys.stderr) + sys.exit(1) def mitmproxy(args=None): # pragma: no cover @@ -76,8 +71,7 @@ def mitmproxy(args=None): # pragma: no cover console_options.limit = args.limit console_options.no_mouse = args.no_mouse - proxy_config = process_options(parser, console_options, args) - server = get_server(console_options.no_server, proxy_config) + server = process_options(parser, console_options, args) m = console.master.ConsoleMaster(server, console_options) except exceptions.OptionsError as e: print("mitmproxy: %s" % e, file=sys.stderr) @@ -103,8 +97,7 @@ def mitmdump(args=None): # pragma: no cover dump_options.flow_detail = args.flow_detail dump_options.keepserving = args.keepserving dump_options.filtstr = " ".join(args.args) if args.args else None - proxy_config = process_options(parser, dump_options, args) - server = get_server(dump_options.no_server, proxy_config) + server = process_options(parser, dump_options, args) master = dump.DumpMaster(server, dump_options) def cleankill(*args, **kwargs): @@ -141,8 +134,7 @@ def mitmweb(args=None): # pragma: no cover web_options.whtpasswd = args.whtpasswd web_options.process_web_options(parser) - proxy_config = process_options(parser, web_options, args) - server = get_server(web_options.no_server, proxy_config) + server = process_options(parser, web_options, args) m = web.master.WebMaster(server, web_options) except exceptions.OptionsError as e: print("mitmweb: %s" % e, file=sys.stderr) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 94e5d573..bdc0db4e 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,104 +1,137 @@ from __future__ import absolute_import, print_function, division - -import contextlib -import blinker -import pprint - -from mitmproxy import exceptions - - -class Options(object): - """ - .changed is a blinker Signal that triggers whenever options are - updated. If any handler in the chain raises an exceptions.OptionsError - exception, all changes are rolled back, the exception is suppressed, - and the .errored signal is notified. - """ - _initialized = False - attributes = [] - - def __new__(cls, *args, **kwargs): - # Initialize instance._opts before __init__ is called. - # This allows us to call super().__init__() last, which then sets - # ._initialized = True as the final operation. - instance = super(Options, cls).__new__(cls) - instance.__dict__["_opts"] = {} - return instance - - def __init__(self): - self.__dict__["changed"] = blinker.Signal() - self.__dict__["errored"] = blinker.Signal() - self.__dict__["_initialized"] = True - - @contextlib.contextmanager - def rollback(self): - old = self._opts.copy() - try: - yield - except exceptions.OptionsError as e: - # Notify error handlers - self.errored.send(self, exc=e) - # Rollback - self.__dict__["_opts"] = old - self.changed.send(self) - - def __eq__(self, other): - return self._opts == other._opts - - def __copy__(self): - return self.__class__(**self._opts) - - def __getattr__(self, attr): - if attr in self._opts: - return self._opts[attr] - else: - raise AttributeError("No such option: %s" % attr) - - def __setattr__(self, attr, value): - if not self._initialized: - self._opts[attr] = value - return - if attr not in self._opts: - raise KeyError("No such option: %s" % attr) - with self.rollback(): - self._opts[attr] = value - self.changed.send(self) - - def get(self, k, d=None): - return self._opts.get(k, d) - - def update(self, **kwargs): - for k in kwargs: - if k not in self._opts: - raise KeyError("No such option: %s" % k) - with self.rollback(): - self._opts.update(kwargs) - self.changed.send(self) - - def setter(self, attr): - """ - Generate a setter for a given attribute. This returns a callable - taking a single argument. - """ - if attr not in self._opts: - raise KeyError("No such option: %s" % attr) - return lambda x: self.__setattr__(attr, x) - - def toggler(self, attr): - """ - Generate a toggler for a boolean attribute. This returns a callable - that takes no arguments. - """ - if attr not in self._opts: - raise KeyError("No such option: %s" % attr) - return lambda: self.__setattr__(attr, not getattr(self, attr)) - - def __repr__(self): - options = pprint.pformat(self._opts, indent=4).strip(" {}") - if "\n" in options: - options = "\n " + options + "\n" - return "{mod}.{cls}({{{options}}})".format( - mod=type(self).__module__, - cls=type(self).__name__, - options=options - ) +from mitmproxy import optmanager +from typing import Tuple, Optional, Sequence # noqa + +APP_HOST = "mitm.it" +APP_PORT = 80 +CA_DIR = "~/.mitmproxy" +LISTEN_PORT = 8080 + +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \ + "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \ + "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \ + "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \ + "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \ + "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \ + "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \ + "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \ + "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + + +class Options(optmanager.OptManager): + def __init__( + self, + # TODO: rename to onboarding_app_* + app=True, # type: bool + app_host=APP_HOST, # type: str + app_port=APP_PORT, # type: int + anticache=False, # type: bool + anticomp=False, # type: bool + client_replay=None, # type: Optional[str] + kill=False, # type: bool + no_server=False, # type: bool + nopop=False, # type: bool + refresh_server_playback=False, # type: bool + rfile=None, # type: Optional[str] + scripts=(), # type: Sequence[str] + showhost=False, # type: bool + replacements=(), # type: Sequence[Tuple[str, str, str]] + rheaders=(), # type: Sequence[str] + setheaders=(), # type: Sequence[Tuple[str, str, str]] + server_replay=None, # type: Optional[str] + stickycookie=None, # type: Optional[str] + stickyauth=None, # type: Optional[str] + stream_large_bodies=None, # type: Optional[str] + verbosity=2, # type: int + outfile=None, # type: Tuple[str, str] + replay_ignore_content=False, # type: bool + replay_ignore_params=(), # type: Sequence[str] + replay_ignore_payload_params=(), # type: Sequence[str] + replay_ignore_host=False, # type: bool + + # Proxy options + auth_nonanonymous=False, # type: bool + auth_singleuser=None, # type: Optional[str] + auth_htpasswd=None, # type: Optional[str] + add_upstream_certs_to_client_chain=False, # type: bool + body_size_limit=None, # type: Optional[int] + cadir = CA_DIR, # type: str + certs = (), # type: Sequence[Tuple[str, str]] + ciphers_client = DEFAULT_CLIENT_CIPHERS, # type: str + ciphers_server = None, # type: Optional[str] + clientcerts = None, # type: Optional[str] + http2 = True, # type: bool + ignore_hosts = (), # type: Sequence[str] + listen_host = "", # type: str + listen_port = LISTEN_PORT, # type: int + mode = "regular", # type: str + no_upstream_cert = False, # type: bool + rawtcp = False, # type: bool + upstream_server = "", # type: str + upstream_auth = "", # type: str + ssl_version_client="secure", # type: str + ssl_version_server="secure", # type: str + ssl_verify_upstream_cert=False, # type: bool + ssl_verify_upstream_trusted_cadir=None, # type: str + ssl_verify_upstream_trusted_ca=None, # type: str + tcp_hosts = (), # type: Sequence[str] + ): + # We could replace all assignments with clever metaprogramming, + # but type hints are a much more valueable asset. + + self.app = app + self.app_host = app_host + self.app_port = app_port + self.anticache = anticache + self.anticomp = anticomp + self.client_replay = client_replay + self.kill = kill + self.no_server = no_server + self.nopop = nopop + self.refresh_server_playback = refresh_server_playback + self.rfile = rfile + self.scripts = scripts + self.showhost = showhost + self.replacements = replacements + self.rheaders = rheaders + self.setheaders = setheaders + self.server_replay = server_replay + self.stickycookie = stickycookie + self.stickyauth = stickyauth + self.stream_large_bodies = stream_large_bodies + self.verbosity = verbosity + self.outfile = outfile + self.replay_ignore_content = replay_ignore_content + self.replay_ignore_params = replay_ignore_params + self.replay_ignore_payload_params = replay_ignore_payload_params + self.replay_ignore_host = replay_ignore_host + + # Proxy options + self.auth_nonanonymous = auth_nonanonymous + self.auth_singleuser = auth_singleuser + self.auth_htpasswd = auth_htpasswd + self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain + self.body_size_limit = body_size_limit + self.cadir = cadir + self.certs = certs + self.ciphers_client = ciphers_client + self.ciphers_server = ciphers_server + self.clientcerts = clientcerts + self.http2 = http2 + self.ignore_hosts = ignore_hosts + self.listen_host = listen_host + self.listen_port = listen_port + self.mode = mode + self.no_upstream_cert = no_upstream_cert + self.rawtcp = rawtcp + self.upstream_server = upstream_server + self.upstream_auth = upstream_auth + self.ssl_version_client = ssl_version_client + self.ssl_version_server = ssl_version_server + self.ssl_verify_upstream_cert = ssl_verify_upstream_cert + self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir + self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca + self.tcp_hosts = tcp_hosts + super(Options, self).__init__() diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py new file mode 100644 index 00000000..e94ef51d --- /dev/null +++ b/mitmproxy/optmanager.py @@ -0,0 +1,108 @@ +from __future__ import absolute_import, print_function, division + +import contextlib +import blinker +import pprint + +from mitmproxy import exceptions + +""" + The base implementation for Options. +""" + + +class OptManager(object): + """ + .changed is a blinker Signal that triggers whenever options are + updated. If any handler in the chain raises an exceptions.OptionsError + exception, all changes are rolled back, the exception is suppressed, + and the .errored signal is notified. + """ + _initialized = False + attributes = [] + + def __new__(cls, *args, **kwargs): + # Initialize instance._opts before __init__ is called. + # This allows us to call super().__init__() last, which then sets + # ._initialized = True as the final operation. + instance = super(OptManager, cls).__new__(cls) + instance.__dict__["_opts"] = {} + return instance + + def __init__(self): + self.__dict__["changed"] = blinker.Signal() + self.__dict__["errored"] = blinker.Signal() + self.__dict__["_initialized"] = True + + @contextlib.contextmanager + def rollback(self): + old = self._opts.copy() + try: + yield + except exceptions.OptionsError as e: + # Notify error handlers + self.errored.send(self, exc=e) + # Rollback + self.__dict__["_opts"] = old + self.changed.send(self) + + def __eq__(self, other): + return self._opts == other._opts + + def __copy__(self): + return self.__class__(**self._opts) + + def __getattr__(self, attr): + if attr in self._opts: + return self._opts[attr] + else: + raise AttributeError("No such option: %s" % attr) + + def __setattr__(self, attr, value): + if not self._initialized: + self._opts[attr] = value + return + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + with self.rollback(): + self._opts[attr] = value + self.changed.send(self) + + def get(self, k, d=None): + return self._opts.get(k, d) + + def update(self, **kwargs): + for k in kwargs: + if k not in self._opts: + raise KeyError("No such option: %s" % k) + with self.rollback(): + self._opts.update(kwargs) + self.changed.send(self) + + def setter(self, attr): + """ + Generate a setter for a given attribute. This returns a callable + taking a single argument. + """ + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + return lambda x: self.__setattr__(attr, x) + + def toggler(self, attr): + """ + Generate a toggler for a boolean attribute. This returns a callable + that takes no arguments. + """ + if attr not in self._opts: + raise KeyError("No such option: %s" % attr) + return lambda: self.__setattr__(attr, not getattr(self, attr)) + + def __repr__(self): + options = pprint.pformat(self._opts, indent=4).strip(" {}") + if "\n" in options: + options = "\n " + options + "\n" + return "{mod}.{cls}({{{options}}})".format( + mod=type(self).__module__, + cls=type(self).__name__, + options=options + ) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index a0d68191..3d384612 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -12,6 +12,7 @@ from mitmproxy import builtins from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import flow +from mitmproxy import options from mitmproxy.web import app from netlib.http import authentication @@ -91,7 +92,7 @@ class WebState(flow.State): ) -class Options(flow.options.Options): +class Options(options.Options): def __init__( self, intercept=None, # type: Optional[str] diff --git a/netlib/encoding.py b/netlib/encoding.py index 8b67b543..e3cf5f30 100644 --- a/netlib/encoding.py +++ b/netlib/encoding.py @@ -12,7 +12,7 @@ from typing import Union # noqa def decode(obj, encoding, errors='strict'): - # type: (Union[str, bytes], str) -> Union[str, bytes] + # type: (Union[str, bytes], str, str) -> Union[str, bytes] """ Decode the given input object @@ -36,7 +36,7 @@ def decode(obj, encoding, errors='strict'): def encode(obj, encoding, errors='strict'): - # type: (Union[str, bytes], str) -> Union[str, bytes] + # type: (Union[str, bytes], str, str) -> Union[str, bytes] """ Encode the given input object diff --git a/release/setup.py b/release/setup.py index 78155140..601654e5 100644 --- a/release/setup.py +++ b/release/setup.py @@ -7,7 +7,7 @@ setup( install_requires=[ "click>=6.2, <7.0", "twine>=1.6.5, <1.7", - "virtualenv>=14.0.5, <14.1", + "virtualenv>=14.0.5, <15.1", "wheel>=0.29.0, <0.30", "six>=1.10.0, <1.11", "pysftp>=0.2.8, !=0.2.9, <0.3", diff --git a/test/mitmproxy/builtins/test_anticache.py b/test/mitmproxy/builtins/test_anticache.py index 127e1c1a..5a00af03 100644 --- a/test/mitmproxy/builtins/test_anticache.py +++ b/test/mitmproxy/builtins/test_anticache.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import anticache from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestAntiCache(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_anticomp.py b/test/mitmproxy/builtins/test_anticomp.py index 601e56c8..6bfd54bb 100644 --- a/test/mitmproxy/builtins/test_anticomp.py +++ b/test/mitmproxy/builtins/test_anticomp.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import anticomp from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestAntiComp(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_filestreamer.py b/test/mitmproxy/builtins/test_filestreamer.py index 002006b7..c1d5947f 100644 --- a/test/mitmproxy/builtins/test_filestreamer.py +++ b/test/mitmproxy/builtins/test_filestreamer.py @@ -7,7 +7,7 @@ import os.path from mitmproxy.builtins import filestreamer from mitmproxy.flow import master, FlowReader from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestStream(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_replace.py b/test/mitmproxy/builtins/test_replace.py index f8010bec..a0b4b722 100644 --- a/test/mitmproxy/builtins/test_replace.py +++ b/test/mitmproxy/builtins/test_replace.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import replace from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestReplace(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py index c9616249..f37c7f94 100644 --- a/test/mitmproxy/builtins/test_script.py +++ b/test/mitmproxy/builtins/test_script.py @@ -4,7 +4,7 @@ from mitmproxy.builtins import script from mitmproxy import exceptions from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options from .. import tutils, mastertest diff --git a/test/mitmproxy/builtins/test_setheaders.py b/test/mitmproxy/builtins/test_setheaders.py index 1a8d048c..4465719d 100644 --- a/test/mitmproxy/builtins/test_setheaders.py +++ b/test/mitmproxy/builtins/test_setheaders.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import setheaders from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestSetHeaders(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_stickyauth.py b/test/mitmproxy/builtins/test_stickyauth.py index 1e617402..9233f435 100644 --- a/test/mitmproxy/builtins/test_stickyauth.py +++ b/test/mitmproxy/builtins/test_stickyauth.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import stickyauth from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options class TestStickyAuth(mastertest.MasterTest): diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py index b8d703bd..81b540db 100644 --- a/test/mitmproxy/builtins/test_stickycookie.py +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -2,7 +2,7 @@ from .. import tutils, mastertest from mitmproxy.builtins import stickycookie from mitmproxy.flow import master from mitmproxy.flow import state -from mitmproxy.flow import options +from mitmproxy import options from netlib import tutils as ntutils diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index e17a125c..36b212a7 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -3,8 +3,7 @@ import io import netlib.utils from netlib.http import Headers -from mitmproxy import filt, controller, flow -from mitmproxy.flow import options +from mitmproxy import filt, controller, flow, options from mitmproxy.contrib import tnetstring from mitmproxy.exceptions import FlowReadException from mitmproxy.models import Error @@ -745,7 +744,7 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] fm = flow.FlowMaster( - flow.options.Options(), + options.Options(), DummyServer(ProxyConfig(options.Options())), s ) @@ -776,7 +775,7 @@ class TestFlowMaster: f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(flow.options.Options(), None, s) + fm = flow.FlowMaster(options.Options(), None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_optmanager.py index af619b27..67f76ecd 100644 --- a/test/mitmproxy/test_options.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,12 +1,12 @@ from __future__ import absolute_import, print_function, division import copy -from mitmproxy import options +from mitmproxy import optmanager from mitmproxy import exceptions from netlib import tutils -class TO(options.Options): +class TO(optmanager.OptManager): def __init__(self, one=None, two=None): self.one = one self.two = two @@ -93,8 +93,8 @@ def test_rollback(): def test_repr(): - assert repr(TO()) == "test.mitmproxy.test_options.TO({'one': None, 'two': None})" - assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_options.TO({ + assert repr(TO()) == "test.mitmproxy.test_optmanager.TO({'one': None, 'two': None})" + assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_optmanager.TO({ 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'two': None })""" diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index a7a3ba3f..afbffb67 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -9,9 +9,8 @@ import traceback import h2 -from mitmproxy.flow import options +from mitmproxy import options from mitmproxy.proxy.config import ProxyConfig -from mitmproxy.cmdline import APP_HOST, APP_PORT import netlib from ..netlib import tservers as netlib_tservers @@ -90,11 +89,11 @@ class _Http2TestBase(object): @classmethod def setup_class(cls): cls.masteroptions = options.Options() - cnf, opts = cls.get_proxy_config() - cls.config = ProxyConfig(opts, **cnf) + opts = cls.get_options() + cls.config = ProxyConfig(opts) tmaster = tservers.TestMaster(opts, cls.config) - tmaster.start_app(APP_HOST, APP_PORT) + tmaster.start_app(options.APP_HOST, options.APP_PORT) cls.proxy = tservers.ProxyThread(tmaster) cls.proxy.start() @@ -103,11 +102,10 @@ class _Http2TestBase(object): cls.proxy.shutdown() @classmethod - def get_proxy_config(cls): + def get_options(cls): opts = options.Options(listen_port=0, no_upstream_cert=False) opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") - d = dict() - return d, opts + return opts @property def master(self): diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 7095d9d2..6e790e28 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -3,10 +3,10 @@ import mock from OpenSSL import SSL from mitmproxy import cmdline +from mitmproxy import options from mitmproxy.proxy import ProxyConfig from mitmproxy.models.connections import ServerConnection from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler -from mitmproxy.flow import options from mitmproxy.proxy import config from netlib.exceptions import TcpDisconnect from pathod import test diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index b8b057fd..233af597 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -486,11 +486,11 @@ class TestSocks5(tservers.SocksModeTest): class TestHttps2Http(tservers.ReverseProxyTest): @classmethod - def get_proxy_config(cls): - d, opts = super(TestHttps2Http, cls).get_proxy_config() + def get_options(cls): + opts = super(TestHttps2Http, cls).get_options() s = parse_server_spec(opts.upstream_server) opts.upstream_server = "http://%s" % s.address - return d, opts + return opts def pathoc(self, ssl, sni=None): """ @@ -766,9 +766,14 @@ class TestFakeResponse(tservers.HTTPProxyTest): class TestServerConnect(tservers.HTTPProxyTest): masterclass = MasterFakeResponse - no_upstream_cert = True ssl = True + @classmethod + def get_options(cls): + opts = tservers.HTTPProxyTest.get_options() + opts.no_upstream_cert = True + return opts + def test_unnecessary_serverconnect(self): """A replayed/fake response with no_upstream_cert should not connect to an upstream server""" assert self.pathod("200").status_code == 200 @@ -1034,20 +1039,34 @@ class AddUpstreamCertsToClientChainMixin: if receivedCert.digest('sha256') == upstreamCert.digest('sha256'): upstream_cert_found_in_client_chain = True break - assert(upstream_cert_found_in_client_chain == self.add_upstream_certs_to_client_chain) - + assert(upstream_cert_found_in_client_chain == self.master.options.add_upstream_certs_to_client_chain) -class TestHTTPSAddUpstreamCertsToClientChainTrue(AddUpstreamCertsToClientChainMixin, tservers.HTTPProxyTest): +class TestHTTPSAddUpstreamCertsToClientChainTrue( + AddUpstreamCertsToClientChainMixin, + tservers.HTTPProxyTest +): """ - If --add-server-certs-to-client-chain is True, then the client should receive the upstream server's certificates + If --add-server-certs-to-client-chain is True, then the client should + receive the upstream server's certificates """ - add_upstream_certs_to_client_chain = True - + @classmethod + def get_options(cls): + opts = super(tservers.HTTPProxyTest, cls).get_options() + opts.add_upstream_certs_to_client_chain = True + return opts -class TestHTTPSAddUpstreamCertsToClientChainFalse(AddUpstreamCertsToClientChainMixin, tservers.HTTPProxyTest): +class TestHTTPSAddUpstreamCertsToClientChainFalse( + AddUpstreamCertsToClientChainMixin, + tservers.HTTPProxyTest +): """ - If --add-server-certs-to-client-chain is False, then the client should not receive the upstream server's certificates + If --add-server-certs-to-client-chain is False, then the client should not + receive the upstream server's certificates """ - add_upstream_certs_to_client_chain = False + @classmethod + def get_options(cls): + opts = super(tservers.HTTPProxyTest, cls).get_options() + opts.add_upstream_certs_to_client_chain = False + return opts diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 495765da..f5119166 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -8,9 +8,7 @@ from mitmproxy.proxy.config import ProxyConfig from mitmproxy.proxy.server import ProxyServer import pathod.test import pathod.pathoc -from mitmproxy import flow, controller -from mitmproxy.flow import options -from mitmproxy.cmdline import APP_HOST, APP_PORT +from mitmproxy import flow, controller, options from mitmproxy import builtins testapp = flask.Flask(__name__) @@ -77,8 +75,8 @@ class ProxyTestBase(object): # Test Configuration ssl = None ssloptions = False - no_upstream_cert = False masterclass = TestMaster + add_upstream_certs_to_client_chain = False @classmethod @@ -90,10 +88,10 @@ class ProxyTestBase(object): ssl=cls.ssl, ssloptions=cls.ssloptions) - cnf, opts = cls.get_proxy_config() - cls.config = ProxyConfig(opts, **cnf) + opts = cls.get_options() + cls.config = ProxyConfig(opts) tmaster = cls.masterclass(opts, cls.config) - tmaster.start_app(APP_HOST, APP_PORT) + tmaster.start_app(options.APP_HOST, options.APP_PORT) cls.proxy = ProxyThread(tmaster) cls.proxy.start() @@ -117,13 +115,11 @@ class ProxyTestBase(object): return self.proxy.tmaster @classmethod - def get_proxy_config(cls): + def get_options(cls): cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") - cnf = dict() - return cnf, options.Options( + return options.Options( listen_port=0, cadir=cls.cadir, - no_upstream_cert = cls.no_upstream_cert, add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain ) @@ -162,11 +158,11 @@ class HTTPProxyTest(ProxyTestBase): p = pathod.pathoc.Pathoc( ("127.0.0.1", self.proxy.port), True, fp=None ) - p.connect((APP_HOST, APP_PORT)) + p.connect((options.APP_HOST, options.APP_PORT)) return p.request("get:'%s'" % page) else: p = self.pathoc() - return p.request("get:'http://%s%s'" % (APP_HOST, page)) + return p.request("get:'http://%s%s'" % (options.APP_HOST, page)) class TResolver: @@ -198,10 +194,10 @@ class TransparentProxyTest(ProxyTestBase): super(TransparentProxyTest, cls).teardown_class() @classmethod - def get_proxy_config(cls): - d, opts = ProxyTestBase.get_proxy_config() + def get_options(cls): + opts = ProxyTestBase.get_options() opts.mode = "transparent" - return d, opts + return opts def pathod(self, spec, sni=None): """ @@ -230,8 +226,8 @@ class ReverseProxyTest(ProxyTestBase): ssl = None @classmethod - def get_proxy_config(cls): - d, opts = ProxyTestBase.get_proxy_config() + def get_options(cls): + opts = ProxyTestBase.get_options() opts.upstream_server = "".join( [ "https" if cls.ssl else "http", @@ -241,7 +237,7 @@ class ReverseProxyTest(ProxyTestBase): ] ) opts.mode = "reverse" - return d, opts + return opts def pathoc(self, sni=None): """ @@ -269,10 +265,10 @@ class ReverseProxyTest(ProxyTestBase): class SocksModeTest(HTTPProxyTest): @classmethod - def get_proxy_config(cls): - d, opts = ProxyTestBase.get_proxy_config() + def get_options(cls): + opts = ProxyTestBase.get_options() opts.mode = "socks5" - return d, opts + return opts class ChainProxyTest(ProxyTestBase): @@ -291,16 +287,16 @@ class ChainProxyTest(ProxyTestBase): cls.chain = [] super(ChainProxyTest, cls).setup_class() for _ in range(cls.n): - cnf, opts = cls.get_proxy_config() - config = ProxyConfig(opts, **cnf) + opts = cls.get_options() + config = ProxyConfig(opts) tmaster = cls.masterclass(opts, config) proxy = ProxyThread(tmaster) proxy.start() cls.chain.insert(0, proxy) # Patch the orginal proxy to upstream mode - cnf, opts = cls.get_proxy_config() - cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts, **cnf) + opts = cls.get_options() + cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts) @classmethod def teardown_class(cls): @@ -315,14 +311,14 @@ class ChainProxyTest(ProxyTestBase): proxy.tmaster.state.clear() @classmethod - def get_proxy_config(cls): - d, opts = super(ChainProxyTest, cls).get_proxy_config() + def get_options(cls): + opts = super(ChainProxyTest, cls).get_options() if cls.chain: # First proxy is in normal mode. opts.update( mode="upstream", upstream_server="http://127.0.0.1:%s" % cls.chain[0].port ) - return d, opts + return opts class HTTPUpstreamProxyTest(ChainProxyTest, HTTPProxyTest): |