diff options
57 files changed, 1410 insertions, 1042 deletions
diff --git a/examples/README b/examples/README index cf5c4d7d..90edf468 100644 --- a/examples/README +++ b/examples/README @@ -7,6 +7,7 @@ add_header.py Simple script that just adds a header to every request change_upstream_proxy.py Dynamically change the upstream proxy dns_spoofing.py Use mitmproxy in a DNS spoofing scenario. dup_and_replay.py Duplicates each request, changes it, and then replays the modified request. +fail_with_500.py Turn every response into an Internal Server Error. filt.py Use mitmproxy's filter expressions in your script. flowwriter.py Only write selected flows into a mitmproxy dumpfile. iframe_injector.py Inject configurable iframe into pages. diff --git a/examples/fail_with_500.py b/examples/fail_with_500.py new file mode 100644 index 00000000..aec85b50 --- /dev/null +++ b/examples/fail_with_500.py @@ -0,0 +1,3 @@ +def response(context, flow): + flow.response.status_code = 500 + flow.response.content = b"" diff --git a/issue_template.md b/issue_template.md index 3f9be788..08d390e4 100644 --- a/issue_template.md +++ b/issue_template.md @@ -10,10 +10,10 @@ ##### What went wrong? -##### Any other comments? +##### Any other comments? What have you tried so far? --- Mitmproxy Version: -Operating System:
\ No newline at end of file +Operating System: diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py index 5b980572..e75aed86 100644 --- a/mitmproxy/console/__init__.py +++ b/mitmproxy/console/__init__.py @@ -19,7 +19,7 @@ from netlib import tcp from .. import flow, script, contentviews from . import flowlist, flowview, help, window, signals, options from . import grideditor, palettes, statusbar, palettepicker -from ..exceptions import FlowReadException +from ..exceptions import FlowReadException, ScriptException EVENTLOG_SIZE = 500 @@ -229,9 +229,10 @@ class ConsoleMaster(flow.FlowMaster): if options.scripts: for i in options.scripts: - err = self.load_script(i) - if err: - print("Script load error: {}".format(err), file=sys.stderr) + try: + self.load_script(i) + except ScriptException as e: + print("Script load error: {}".format(e), file=sys.stderr) sys.exit(1) if options.outfile: @@ -320,11 +321,11 @@ class ConsoleMaster(flow.FlowMaster): try: s = script.Script(command, script.ScriptContext(self)) s.load() - except script.ScriptException as v: + except script.ScriptException as e: signals.status_message.send( - message = "Error loading script." + message='Error loading "{}".'.format(command) ) - signals.add_event("Error loading script:\n%s" % v.args[0], "error") + signals.add_event('Error loading "{}":\n{}'.format(command, e), "error") return if f.request: @@ -336,13 +337,6 @@ class ConsoleMaster(flow.FlowMaster): s.unload() signals.flow_change.send(self, flow = f) - def set_script(self, command): - if not command: - return - ret = self.load_script(command) - if ret: - signals.status_message.send(message=ret) - def toggle_eventlog(self): self.eventlog = not self.eventlog signals.pop_view_state.send(self) @@ -670,7 +664,13 @@ class ConsoleMaster(flow.FlowMaster): self.unload_scripts() for command in commands: - self.load_script(command) + try: + self.load_script(command) + except ScriptException as e: + signals.status_message.send( + message='Error loading "{}".'.format(command) + ) + signals.add_event('Error loading "{}":\n{}'.format(command, e), "error") signals.update_settings.send(self) def stop_client_playback_prompt(self, a): diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index 4e472fb6..25658dfa 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -154,7 +154,7 @@ def raw_format_flow(f, focus, extended): if f["intercepted"] and not f["acked"]: uc = "intercept" - elif f["resp_code"] or f["err_msg"]: + elif "resp_code" in f or "err_msg" in f: uc = "text" else: uc = "title" @@ -173,7 +173,7 @@ def raw_format_flow(f, focus, extended): ("fixed", preamble, urwid.Text("")) ) - if f["resp_code"]: + if "resp_code" in f: codes = { 2: "code_200", 3: "code_300", @@ -185,6 +185,8 @@ def raw_format_flow(f, focus, extended): if f["resp_is_replay"]: resp.append(fcol(SYMBOL_REPLAY, "replay")) resp.append(fcol(f["resp_code"], ccol)) + if extended: + resp.append(fcol(f["resp_reason"], ccol)) if f["intercepted"] and f["resp_code"] and not f["acked"]: rc = "intercept" else: @@ -412,7 +414,6 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False): req_http_version = f.request.http_version, err_msg = f.error.msg if f.error else None, - resp_code = f.response.status_code if f.response else None, marked = marked, ) @@ -430,6 +431,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False): d.update(dict( resp_code = f.response.status_code, + resp_reason = f.response.reason, resp_is_replay = f.response.is_replay, resp_clen = contentdesc, roundtrip = roundtrip, diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index b761a924..b2ebe49e 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -364,12 +364,11 @@ class FlowView(tabs.Tabs): self.edit_form(conn) def set_cookies(self, lst, conn): - od = odict.ODict(lst) - conn.set_cookies(od) + conn.cookies = odict.ODict(lst) signals.flow_change.send(self, flow = self.flow) def set_setcookies(self, data, conn): - conn.set_cookies(data) + conn.cookies = data signals.flow_change.send(self, flow = self.flow) def edit(self, part): @@ -389,7 +388,7 @@ class FlowView(tabs.Tabs): self.master.view_grideditor( grideditor.CookieEditor( self.master, - message.get_cookies().lst, + message.cookies.lst, self.set_cookies, message ) @@ -398,7 +397,7 @@ class FlowView(tabs.Tabs): self.master.view_grideditor( grideditor.SetCookieEditor( self.master, - message.get_cookies(), + message.cookies, self.set_setcookies, message ) diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py index 597a7e7a..46ff348e 100644 --- a/mitmproxy/console/grideditor.py +++ b/mitmproxy/console/grideditor.py @@ -642,8 +642,8 @@ class ScriptEditor(GridEditor): def is_error(self, col, val): try: script.Script.parse_command(val) - except script.ScriptException as v: - return str(v) + except script.ScriptException as e: + return str(e) class HostPatternEditor(GridEditor): diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index c308adf0..aae397cd 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -5,9 +5,9 @@ import click import itertools from netlib import tcp -import netlib.utils +from netlib.utils import bytes_to_escaped_str, pretty_size from . import flow, filt, contentviews -from .exceptions import ContentViewException, FlowReadException +from .exceptions import ContentViewException, FlowReadException, ScriptException class DumpError(Exception): @@ -125,9 +125,10 @@ class DumpMaster(flow.FlowMaster): scripts = options.scripts or [] for command in scripts: - err = self.load_script(command, use_reloader=True) - if err: - raise DumpError(err) + try: + self.load_script(command, use_reloader=True) + except ScriptException as e: + raise DumpError(str(e)) if options.rfile: try: @@ -174,8 +175,8 @@ class DumpMaster(flow.FlowMaster): if self.o.flow_detail >= 2: headers = "\r\n".join( "{}: {}".format( - click.style(k, fg="blue", bold=True), - click.style(v, fg="blue")) + click.style(bytes_to_escaped_str(k), fg="blue", bold=True), + click.style(bytes_to_escaped_str(v), fg="blue")) for k, v in message.headers.fields ) self.echo(headers, indent=4) @@ -237,7 +238,7 @@ class DumpMaster(flow.FlowMaster): stickycookie = "" if flow.client_conn: - client = click.style(flow.client_conn.address.host, bold=True) + client = click.style(bytes_to_escaped_str(flow.client_conn.address.host), bold=True) else: client = click.style("[replay]", fg="yellow", bold=True) @@ -246,12 +247,12 @@ class DumpMaster(flow.FlowMaster): GET="green", DELETE="red" ).get(method.upper(), "magenta") - method = click.style(method, fg=method_color, bold=True) + method = click.style(bytes_to_escaped_str(method), fg=method_color, bold=True) if self.showhost: url = flow.request.pretty_url else: url = flow.request.url - url = click.style(url, bold=True) + url = click.style(bytes_to_escaped_str(url), bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): @@ -281,12 +282,12 @@ class DumpMaster(flow.FlowMaster): elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(flow.response.reason, fg=code_color, bold=True) + reason = click.style(bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" else: - size = netlib.utils.pretty_size(len(flow.response.content)) + size = pretty_size(len(flow.response.content)) size = click.style(size, bold=True) arrows = click.style("<<", bold=True) @@ -346,5 +347,6 @@ class DumpMaster(flow.FlowMaster): def run(self): # pragma: no cover if self.o.rfile and not self.o.keepserving: + self.unload_scripts() # make sure to trigger script unload events. return - super(DumpMaster, self).run()
\ No newline at end of file + super(DumpMaster, self).run() diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 86bf75ae..8f989063 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -7,6 +7,10 @@ See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/ """ from __future__ import (absolute_import, print_function, division) +import traceback + +import sys + class ProxyException(Exception): """ @@ -46,6 +50,10 @@ class HttpProtocolException(ProtocolException): pass +class Http2ProtocolException(ProtocolException): + pass + + class ServerException(ProxyException): pass @@ -59,8 +67,24 @@ class ReplayException(ProxyException): class ScriptException(ProxyException): - pass + @classmethod + def from_exception_context(cls, cut_tb=1): + """ + Must be called while the current stack handles an exception. + + Args: + cut_tb: remove N frames from the stack trace to hide internal calls. + """ + exc_type, exc_value, exc_traceback = sys.exc_info() + + while cut_tb > 0: + exc_traceback = exc_traceback.tb_next + cut_tb -= 1 + + tb = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback)) + + return cls(tb) class FlowReadException(ProxyException): - pass
\ No newline at end of file + pass diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index d05aabbb..ccedd1d4 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -13,9 +13,11 @@ from six.moves import http_cookies, http_cookiejar, urllib import os import re -from netlib import wsgi +from typing import List, Optional, Set + +from netlib import wsgi, odict from netlib.exceptions import HttpException -from netlib.http import Headers, http1 +from netlib.http import Headers, http1, cookies from . import controller, tnetstring, filt, script, version, flow_format_compat from .onboarding import app from .proxy.config import HostMatcher @@ -171,7 +173,7 @@ class StreamLargeBodies(object): expected_size = http1.expected_http_body_size( flow.request, flow.response if not is_request else None ) - if not (0 <= expected_size <= self.max_size): + if not r.content and not (0 <= expected_size <= self.max_size): # r.stream may already be a callable, which we want to preserve. r.stream = r.stream or True @@ -313,15 +315,17 @@ class StickyCookieState: self.jar = defaultdict(dict) self.flt = flt - def ckey(self, m, f): + def ckey(self, attrs, f): """ Returns a (domain, port, path) tuple. """ - return ( - m["domain"] or f.request.host, - f.request.port, - m["path"] or "/" - ) + domain = f.request.host + path = "/" + if attrs["domain"]: + domain = attrs["domain"][-1] + if attrs["path"]: + path = attrs["path"][-1] + return (domain, f.request.port, path) def domain_match(self, a, b): if http_cookiejar.domain_match(a, b): @@ -334,11 +338,12 @@ class StickyCookieState: for i in f.response.headers.get_all("set-cookie"): # FIXME: We now know that Cookie.py screws up some cookies with # valid RFC 822/1123 datetime specifications for expiry. Sigh. - c = http_cookies.SimpleCookie(str(i)) - for m in c.values(): - k = self.ckey(m, f) - if self.domain_match(f.request.host, k[0]): - self.jar[k][m.key] = m + name, value, attrs = cookies.parse_set_cookie_header(str(i)) + a = self.ckey(attrs, f) + if self.domain_match(f.request.host, a[0]): + b = attrs.lst + b.insert(0, [name, value]) + self.jar[a][name] = odict.ODictCaseless(b) def handle_request(self, f): l = [] @@ -350,7 +355,8 @@ class StickyCookieState: f.request.path.startswith(i[2]) ] if all(match): - l.extend([m.output(header="").strip() for m in self.jar[i].values()]) + c = self.jar[i] + l.extend([cookies.format_cookie_header(c[name]) for name in c.keys()]) if l: f.request.stickycookie = True f.request.headers["cookie"] = "; ".join(l) @@ -374,8 +380,11 @@ class StickyAuthState: f.request.headers["authorization"] = self.hosts[host] +@six.add_metaclass(ABCMeta) class FlowList(object): - __metaclass__ = ABCMeta + + def __init__(self): + self._list = [] # type: List[Flow] def __iter__(self): return iter(self._list) @@ -414,7 +423,7 @@ class FlowList(object): class FlowView(FlowList): def __init__(self, store, filt=None): - self._list = [] + super(FlowView, self).__init__() if not filt: filt = lambda flow: True self._build(store, filt) @@ -456,7 +465,7 @@ class FlowStore(FlowList): """ def __init__(self): - self._list = [] + super(FlowStore, self).__init__() self._set = set() # Used for O(1) lookups self.views = [] self._recalculate_views() @@ -647,18 +656,18 @@ class FlowMaster(controller.ServerMaster): self.server_playback = None self.client_playback = None self.kill_nonreplay = False - self.scripts = [] + self.scripts = [] # type: List[script.Script] self.pause_scripts = False - self.stickycookie_state = False + self.stickycookie_state = None # type: Optional[StickyCookieState] self.stickycookie_txt = None - self.stickyauth_state = False + self.stickyauth_state = False # type: Optional[StickyAuthState] self.stickyauth_txt = None self.anticache = False self.anticomp = False - self.stream_large_bodies = False + self.stream_large_bodies = None # type: Optional[StreamLargeBodies] self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() @@ -695,14 +704,13 @@ class FlowMaster(controller.ServerMaster): def load_script(self, command, use_reloader=False): """ - Loads a script. Returns an error description if something went - wrong. + Loads a script. + + Raises: + ScriptException """ - try: - s = script.Script(command, script.ScriptContext(self)) - s.load() - except script.ScriptException as e: - return traceback.format_exc(e) + s = script.Script(command, script.ScriptContext(self)) + s.load() if use_reloader: script.reloader.watch(s, lambda: self.event_queue.put(("script_change", s))) self.scripts.append(s) @@ -712,7 +720,7 @@ class FlowMaster(controller.ServerMaster): try: script_obj.run(name, *args, **kwargs) except script.ScriptException as e: - self.add_event("Script error:\n" + str(e), "error") + self.add_event("Script error:\n{}".format(e), "error") def run_script_hook(self, name, *args, **kwargs): for script_obj in self.scripts: @@ -1021,8 +1029,6 @@ class FlowMaster(controller.ServerMaster): return f def handle_responseheaders(self, f): - self.run_script_hook("responseheaders", f) - try: if self.stream_large_bodies: self.stream_large_bodies.run(f, False) @@ -1030,6 +1036,8 @@ class FlowMaster(controller.ServerMaster): f.reply(Kill) return + self.run_script_hook("responseheaders", f) + f.reply() return f @@ -1069,12 +1077,12 @@ class FlowMaster(controller.ServerMaster): s.unload() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') + self.add_event('Error reloading "{}":\n{}'.format(s.filename, e), 'error') try: s.load() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') + self.add_event('Error reloading "{}":\n{}'.format(s.filename, e), 'error') else: self.add_event('"{}" reloaded.'.format(s.filename), 'info') return ok diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index 5c6952f1..d9111303 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -11,7 +11,7 @@ from netlib.http import Headers from h2.exceptions import H2Error from .. import utils -from ..exceptions import HttpProtocolException, ProtocolException +from ..exceptions import HttpProtocolException, Http2ProtocolException, ProtocolException from ..models import ( HTTPFlow, HTTPResponse, @@ -238,7 +238,7 @@ class HttpLayer(Layer): try: response = make_error_response(code, message) self.send_response(response) - except (NetlibException, H2Error): + except (NetlibException, H2Error, Http2ProtocolException): self.log(traceback.format_exc(), "debug") def change_upstream_proxy_server(self, address): @@ -283,9 +283,9 @@ class HttpLayer(Layer): try: get_response() - except NetlibException as v: + except NetlibException as e: self.log( - "server communication error: %s" % repr(v), + "server communication error: %s" % repr(e), level="debug" ) # In any case, we try to reconnect at least once. This is @@ -299,6 +299,11 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream + + if isinstance(e, Http2ProtocolException): + # do not try to reconnect for HTTP2 + raise ProtocolException("First and only attempt to get response via HTTP2 failed.") + self.disconnect() self.connect() get_response() diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 03d4aefc..1cc12792 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -4,18 +4,20 @@ import threading import time from six.moves import queue -import h2 +import traceback import six from h2.connection import H2Connection +from h2.exceptions import StreamClosedError +from h2 import events from netlib.tcp import ssl_read_select from netlib.exceptions import HttpException from netlib.http import Headers -from netlib.utils import http2_read_raw_frame +from netlib.utils import http2_read_raw_frame, parse_url from .base import Layer from .http import _HttpTransmissionLayer, HttpLayer -from .. import utils +from ..exceptions import ProtocolException, Http2ProtocolException from ..models import HTTPRequest, HTTPResponse @@ -26,11 +28,6 @@ class SafeH2Connection(H2Connection): self.conn = conn self.lock = threading.RLock() - def safe_close_connection(self, error_code): - with self.lock: - self.close_connection(error_code) - self.conn.send(self.data_to_send()) - def safe_increment_flow_control(self, stream_id, length): if length == 0: return @@ -47,7 +44,7 @@ class SafeH2Connection(H2Connection): with self.lock: try: self.reset_stream(stream_id, error_code) - except h2.exceptions.StreamClosedError: + except StreamClosedError: # pragma: no cover # stream is already closed - good pass self.conn.send(self.data_to_send()) @@ -59,9 +56,9 @@ class SafeH2Connection(H2Connection): def safe_send_headers(self, is_zombie, stream_id, headers): with self.lock: - if is_zombie(): - return - self.send_headers(stream_id, headers) + if is_zombie(): # pragma: no cover + raise Http2ProtocolException("Zombie Stream") + self.send_headers(stream_id, headers.fields) self.conn.send(self.data_to_send()) def safe_send_body(self, is_zombie, stream_id, chunks): @@ -69,9 +66,9 @@ class SafeH2Connection(H2Connection): position = 0 while position < len(chunk): self.lock.acquire() - if is_zombie(): + if is_zombie(): # pragma: no cover self.lock.release() - return + raise Http2ProtocolException("Zombie Stream") max_outbound_frame_size = self.max_outbound_frame_size frame_chunk = chunk[position:position + max_outbound_frame_size] if self.local_flow_control_window(stream_id) < len(frame_chunk): @@ -83,8 +80,8 @@ class SafeH2Connection(H2Connection): self.lock.release() position += max_outbound_frame_size with self.lock: - if is_zombie(): - return + if is_zombie(): # pragma: no cover + raise Http2ProtocolException("Zombie Stream") self.end_stream(stream_id) self.conn.send(self.data_to_send()) @@ -95,32 +92,27 @@ class Http2Layer(Layer): super(Http2Layer, self).__init__(ctx) self.mode = mode self.streams = dict() - self.client_reset_streams = [] - self.server_reset_streams = [] self.server_to_client_stream_ids = dict([(0, 0)]) - self.client_conn.h2 = SafeH2Connection(self.client_conn, client_side=False) + self.client_conn.h2 = SafeH2Connection(self.client_conn, client_side=False, header_encoding=False) # make sure that we only pass actual SSL.Connection objects in here, # because otherwise ssl_read_select fails! self.active_conns = [self.client_conn.connection] def _initiate_server_conn(self): - self.server_conn.h2 = SafeH2Connection(self.server_conn, client_side=True) + self.server_conn.h2 = SafeH2Connection(self.server_conn, client_side=True, header_encoding=False) self.server_conn.h2.initiate_connection() self.server_conn.send(self.server_conn.h2.data_to_send()) self.active_conns.append(self.server_conn.connection) def connect(self): # pragma: no cover - raise ValueError("CONNECT inside an HTTP2 stream is not supported.") - # self.ctx.connect() - # self.server_conn.connect() - # self._initiate_server_conn() + raise Http2ProtocolException("HTTP2 layer should already have a connection.") def set_server(self): # pragma: no cover - raise NotImplementedError("Cannot change server for HTTP2 connections.") + raise Http2ProtocolException("Cannot change server for HTTP2 connections.") def disconnect(self): # pragma: no cover - raise NotImplementedError("Cannot dis- or reconnect in HTTP2 connections.") + raise Http2ProtocolException("Cannot dis- or reconnect in HTTP2 connections.") def next_layer(self): # pragma: no cover # WebSockets over HTTP/2? @@ -140,31 +132,28 @@ class Http2Layer(Layer): else: eid = event.stream_id - if isinstance(event, h2.events.RequestReceived): - headers = Headers([[str(k), str(v)] for k, v in event.headers]) + if isinstance(event, events.RequestReceived): + headers = Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) self.streams[eid].timestamp_start = time.time() self.streams[eid].start() - elif isinstance(event, h2.events.ResponseReceived): - headers = Headers([[str(k), str(v)] for k, v in event.headers]) + elif isinstance(event, events.ResponseReceived): + headers = Headers([[k, v] for k, v in event.headers]) self.streams[eid].queued_data_length = 0 self.streams[eid].timestamp_start = time.time() self.streams[eid].response_headers = headers self.streams[eid].response_arrived.set() - elif isinstance(event, h2.events.DataReceived): + elif isinstance(event, events.DataReceived): if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit: raise HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit)) self.streams[eid].data_queue.put(event.data) self.streams[eid].queued_data_length += len(event.data) source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) - elif isinstance(event, h2.events.StreamEnded): + elif isinstance(event, events.StreamEnded): self.streams[eid].timestamp_end = time.time() self.streams[eid].data_finished.set() - elif isinstance(event, h2.events.StreamReset): + elif isinstance(event, events.StreamReset): self.streams[eid].zombie = time.time() - self.client_reset_streams.append(self.streams[eid].client_stream_id) - if self.streams[eid].server_stream_id: - self.server_reset_streams.append(self.streams[eid].server_stream_id) if eid in self.streams and event.error_code == 0x8: if is_server: other_stream_id = self.streams[eid].client_stream_id @@ -172,14 +161,14 @@ class Http2Layer(Layer): other_stream_id = self.streams[eid].server_stream_id if other_stream_id is not None: other_conn.h2.safe_reset_stream(other_stream_id, event.error_code) - elif isinstance(event, h2.events.RemoteSettingsChanged): + elif isinstance(event, events.RemoteSettingsChanged): new_settings = dict([(id, cs.new_value) for (id, cs) in six.iteritems(event.changed_settings)]) other_conn.h2.safe_update_settings(new_settings) - elif isinstance(event, h2.events.ConnectionTerminated): + elif isinstance(event, events.ConnectionTerminated): # Do not immediately terminate the other connection. # Some streams might be still sending data to the client. return False - elif isinstance(event, h2.events.PushedStreamReceived): + elif isinstance(event, events.PushedStreamReceived): # pushed stream ids should be uniq and not dependent on race conditions # only the parent stream id must be looked up first parent_eid = self.server_to_client_stream_ids[event.parent_stream_id] @@ -195,7 +184,7 @@ class Http2Layer(Layer): self.streams[event.pushed_stream_id].timestamp_end = time.time() self.streams[event.pushed_stream_id].request_data_finished.set() self.streams[event.pushed_stream_id].start() - elif isinstance(event, h2.events.TrailersReceived): + elif isinstance(event, events.TrailersReceived): raise NotImplementedError() return True @@ -227,14 +216,16 @@ class Http2Layer(Layer): try: raw_frame = b''.join(http2_read_raw_frame(source_conn.rfile)) except: + # read frame failed: connection closed + # kill all streams for stream in self.streams.values(): stream.zombie = time.time() return - events = source_conn.h2.receive_data(raw_frame) + incoming_events = source_conn.h2.receive_data(raw_frame) source_conn.send(source_conn.h2.data_to_send()) - for event in events: + for event in incoming_events: if not self._handle_event(event, source_conn, other_conn, is_server): return @@ -244,7 +235,7 @@ class Http2Layer(Layer): class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): def __init__(self, ctx, stream_id, request_headers): - super(Http2SingleStreamLayer, self).__init__(ctx) + super(Http2SingleStreamLayer, self).__init__(ctx, name="Thread-Http2SingleStreamLayer-{}".format(stream_id)) self.zombie = None self.client_stream_id = stream_id self.server_stream_id = None @@ -284,10 +275,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): @queued_data_length.setter def queued_data_length(self, v): - if self.response_arrived.is_set(): - return self.response_queued_data_length - else: - return self.request_queued_data_length + self.request_queued_data_length = v def is_zombie(self): return self.zombie is not None @@ -309,7 +297,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): else: # pragma: no cover first_line_format = "absolute" # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = utils.parse_url(path) + scheme, host, port, _ = parse_url(path) if authority: host, _, port = authority.partition(':') @@ -339,6 +327,9 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): timestamp_end=self.timestamp_end, ) + def read_request_body(self, request): # pragma: no cover + raise NotImplementedError() + def send_request(self, message): if self.pushed: # nothing to do here @@ -346,8 +337,8 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): with self.server_conn.h2.lock: # We must not assign a stream id if we are already a zombie. - if self.zombie: - return + if self.zombie: # pragma: no cover + raise Http2ProtocolException("Zombie Stream") self.server_stream_id = self.server_conn.h2.get_next_available_stream_id() self.server_to_client_stream_ids[self.server_stream_id] = self.client_stream_id @@ -362,6 +353,8 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): self.server_stream_id, message.body ) + if self.zombie: # pragma: no cover + raise Http2ProtocolException("Zombie Stream") def read_response_headers(self): self.response_arrived.wait() @@ -388,8 +381,8 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): while self.response_data_queue.qsize() > 0: yield self.response_data_queue.get() return - if self.zombie: - return + if self.zombie: # pragma: no cover + raise Http2ProtocolException("Zombie Stream") def send_response_headers(self, response): self.client_conn.h2.safe_send_headers( @@ -397,6 +390,8 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): self.client_stream_id, response.headers ) + if self.zombie: # pragma: no cover + raise Http2ProtocolException("Zombie Stream") def send_response_body(self, _response, chunks): self.client_conn.h2.safe_send_body( @@ -404,20 +399,28 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): self.client_stream_id, chunks ) + if self.zombie: # pragma: no cover + raise Http2ProtocolException("Zombie Stream") def check_close_connection(self, flow): # This layer only handles a single stream. # RFC 7540 8.1: An HTTP request/response exchange fully consumes a single stream. return True - def connect(self): # pragma: no cover - raise ValueError("CONNECT inside an HTTP2 stream is not supported.") - def set_server(self, *args, **kwargs): # pragma: no cover # do not mess with the server connection - all streams share it. pass def run(self): + self() + + def __call__(self): layer = HttpLayer(self, self.mode) - layer() + + try: + layer() + except ProtocolException as e: + self.log(repr(e), "info") + self.log(traceback.format_exc(), "debug") + self.zombie = time.time() diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 4304bd0b..8483d3df 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -36,7 +36,7 @@ class ProxyServer(tcp.TCPServer): def __init__(self, config): """ - Raises ProxyServerError if there's a startup problem. + Raises ServerException if there's a startup problem. """ self.config = config try: diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py index 5a8334c4..484025b4 100644 --- a/mitmproxy/script/script.py +++ b/mitmproxy/script/script.py @@ -74,18 +74,20 @@ class Script(object): script_dir = os.path.dirname(os.path.abspath(self.args[0])) self.ns = {'__file__': os.path.abspath(self.args[0])} sys.path.append(script_dir) + sys.path.append(os.path.join(script_dir, "..")) try: with open(self.filename) as f: code = compile(f.read(), self.filename, 'exec') exec (code, self.ns, self.ns) - except Exception as e: + except Exception: six.reraise( ScriptException, - ScriptException(str(e)), + ScriptException.from_exception_context(), sys.exc_info()[2] ) finally: sys.path.pop() + sys.path.pop() return self.run("start", self.args) def unload(self): @@ -111,10 +113,10 @@ class Script(object): if f: try: return f(self.ctx, *args, **kwargs) - except Exception as e: + except Exception: six.reraise( ScriptException, - ScriptException(str(e)), + ScriptException.from_exception_context(), sys.exc_info()[2] ) else: diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py index 5fd062ea..cda5bba6 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils.py @@ -7,6 +7,9 @@ import json import importlib import inspect +import netlib.utils + + def timestamp(): """ Returns a serializable UTC timestamp. @@ -73,25 +76,7 @@ def pretty_duration(secs): return "{:.0f}ms".format(secs * 1000) -class Data: - - def __init__(self, name): - m = importlib.import_module(name) - dirname = os.path.dirname(inspect.getsourcefile(m)) - self.dirname = os.path.abspath(dirname) - - def path(self, path): - """ - Returns a path to the package data housed at 'path' under this - module.Path can be a path to a file, or to a directory. - - This function will raise ValueError if the path does not exist. - """ - fullpath = os.path.join(self.dirname, path) - if not os.path.exists(fullpath): - raise ValueError("dataPath: %s does not exist." % fullpath) - return fullpath -pkg_data = Data(__name__) +pkg_data = netlib.utils.Data(__name__) class LRUCache: diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 63f60a8d..da1e7229 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -1,3 +1,6 @@ from __future__ import (absolute_import, print_function, division) -from netlib.version import *
\ No newline at end of file +from netlib.version import VERSION, IVERSION + +NAME = "mitmproxy" +NAMEVERSION = NAME + " " + VERSION diff --git a/mitmproxy/web/__init__.py b/mitmproxy/web/__init__.py index 62468d95..956d221d 100644 --- a/mitmproxy/web/__init__.py +++ b/mitmproxy/web/__init__.py @@ -184,6 +184,8 @@ class WebMaster(flow.FlowMaster): iol.add_callback(self.start) tornado.ioloop.PeriodicCallback(lambda: self.tick(timeout=0), 5).start() try: + print("Server listening at http://{}:{}".format( + self.options.wiface, self.options.wport), file=sys.stderr) iol.start() except (Stop, KeyboardInterrupt): self.shutdown() diff --git a/mitmproxy/web/static/app.css b/mitmproxy/web/static/app.css index 90b177c2..824dd827 100644 --- a/mitmproxy/web/static/app.css +++ b/mitmproxy/web/static/app.css @@ -163,6 +163,9 @@ header .menu { max-height: 500px; overflow-y: auto; } +.menu .btn { + margin: 2px 2px 2px 2px; +} .flow-table { width: 100%; overflow-y: scroll; diff --git a/mitmproxy/web/static/app.js b/mitmproxy/web/static/app.js index 2dadc696..1e8e313d 100644 --- a/mitmproxy/web/static/app.js +++ b/mitmproxy/web/static/app.js @@ -481,7 +481,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de Object.defineProperty(exports, "__esModule", { value: true }); -exports.Splitter = exports.Router = undefined; +exports.ToggleComponent = exports.Splitter = exports.Router = undefined; var _react = require("react"); @@ -631,6 +631,27 @@ var Splitter = exports.Splitter = _react2.default.createClass({ } }); +var ToggleComponent = exports.ToggleComponent = function ToggleComponent(props) { + return _react2.default.createElement( + "div", + { + className: "btn " + (props.checked ? "btn-primary" : "btn-default"), + onClick: props.onToggleChanged }, + _react2.default.createElement( + "span", + null, + _react2.default.createElement("i", { className: "fa " + (props.checked ? "fa-check-square-o" : "fa-square-o") }), + " ", + props.name + ) + ); +}; + +ToggleComponent.propTypes = { + name: _react2.default.PropTypes.string.isRequired, + onToggleChanged: _react2.default.PropTypes.func.isRequired +}; + },{"lodash":"lodash","react":"react","react-dom":"react-dom"}],5:[function(require,module,exports){ "use strict"; @@ -2912,6 +2933,8 @@ Object.defineProperty(exports, "__esModule", { }); exports.Header = exports.MainMenu = undefined; +var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }(); + var _react = require("react"); var _react2 = _interopRequireDefault(_react); @@ -2936,6 +2959,12 @@ var _actions = require("../actions.js"); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } +function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } + +function _possibleConstructorReturn(self, call) { if (!self) { throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); } return call && (typeof call === "object" || typeof call === "function") ? call : self; } + +function _inherits(subClass, superClass) { if (typeof superClass !== "function" && superClass !== null) { throw new TypeError("Super expression must either be null or a function, not " + typeof superClass); } subClass.prototype = Object.create(superClass && superClass.prototype, { constructor: { value: subClass, enumerable: false, writable: true, configurable: true } }); if (superClass) Object.setPrototypeOf ? Object.setPrototypeOf(subClass, superClass) : subClass.__proto__ = superClass; } + var FilterDocs = _react2.default.createClass({ displayName: "FilterDocs", @@ -3191,7 +3220,6 @@ var ViewMenu = _react2.default.createClass({ mixins: [_common.Router], toggleEventLog: function toggleEventLog() { var d = {}; - if (this.getQuery()[_actions.Query.SHOW_EVENTLOG]) { d[_actions.Query.SHOW_EVENTLOG] = undefined; } else { @@ -3199,29 +3227,69 @@ var ViewMenu = _react2.default.createClass({ } this.updateLocation(undefined, d); + console.log('toggleevent'); }, render: function render() { var showEventLog = this.getQuery()[_actions.Query.SHOW_EVENTLOG]; return _react2.default.createElement( "div", null, - _react2.default.createElement( - "button", - { - className: "btn " + (showEventLog ? "btn-primary" : "btn-default"), - onClick: this.toggleEventLog }, - _react2.default.createElement("i", { className: "fa fa-database" }), - " Show Eventlog" - ), - _react2.default.createElement( - "span", - null, - " " - ) + _react2.default.createElement(_common.ToggleComponent, { + checked: showEventLog, + name: "Show Eventlog", + onToggleChanged: this.toggleEventLog }) ); } }); +var OptionMenu = function (_React$Component) { + _inherits(OptionMenu, _React$Component); + + function OptionMenu(props) { + _classCallCheck(this, OptionMenu); + + var _this = _possibleConstructorReturn(this, Object.getPrototypeOf(OptionMenu).call(this, props)); + + _this.state = { + options: [{ name: "--host", checked: true }, { name: "--no-upstream-cert", checked: false }, { name: "--http2", checked: false }, { name: "--anticache", checked: false }, { name: "--anticomp", checked: false }, { name: "--stickycookie", checked: true }, { name: "--stickyauth", checked: false }, { name: "--stream", checked: false }] + }; + return _this; + } + + _createClass(OptionMenu, [{ + key: "setOption", + value: function setOption(entry) { + console.log(entry.name); //TODO: get options from outside and remove state + entry.checked = !entry.checked; + this.setState({ options: this.state.options }); + } + }, { + key: "render", + value: function render() { + var _this2 = this; + + return _react2.default.createElement( + "div", + null, + this.state.options.map(function (entry, i) { + return _react2.default.createElement(_common.ToggleComponent, { + key: i, + checked: entry.checked, + name: entry.name, + onToggleChanged: function onToggleChanged() { + return _this2.setOption(entry); + } }); + }) + ); + } + }]); + + return OptionMenu; +}(_react2.default.Component); + +OptionMenu.title = "Options"; + + var ReportsMenu = _react2.default.createClass({ displayName: "ReportsMenu", @@ -3318,7 +3386,7 @@ var FileMenu = _react2.default.createClass({ } }); -var header_entries = [MainMenu, ViewMenu /*, ReportsMenu */]; +var header_entries = [MainMenu, ViewMenu, OptionMenu /*, ReportsMenu */]; var Header = exports.Header = _react2.default.createClass({ displayName: "Header", diff --git a/mitmproxy/web/static/vendor.js b/mitmproxy/web/static/vendor.js index 548fc355..44bc86d8 100644 --- a/mitmproxy/web/static/vendor.js +++ b/mitmproxy/web/static/vendor.js @@ -2116,13 +2116,15 @@ var KNOWN_STATICS = { }; module.exports = function hoistNonReactStatics(targetComponent, sourceComponent) { - var keys = Object.getOwnPropertyNames(sourceComponent); - for (var i=0; i<keys.length; ++i) { - if (!REACT_STATICS[keys[i]] && !KNOWN_STATICS[keys[i]]) { - try { - targetComponent[keys[i]] = sourceComponent[keys[i]]; - } catch (error) { - + if (typeof sourceComponent !== 'string') { // don't hoist over string (html) components + var keys = Object.getOwnPropertyNames(sourceComponent); + for (var i=0; i<keys.length; ++i) { + if (!REACT_STATICS[keys[i]] && !KNOWN_STATICS[keys[i]]) { + try { + targetComponent[keys[i]] = sourceComponent[keys[i]]; + } catch (error) { + + } } } } @@ -2991,8 +2993,8 @@ function keysIn(object) { module.exports = keys; },{"lodash._getnative":24,"lodash.isarguments":25,"lodash.isarray":26}],28:[function(require,module,exports){ -/* eslint-disable no-unused-vars */ 'use strict'; +/* eslint-disable no-unused-vars */ var hasOwnProperty = Object.prototype.hasOwnProperty; var propIsEnumerable = Object.prototype.propertyIsEnumerable; @@ -3004,7 +3006,51 @@ function toObject(val) { return Object(val); } -module.exports = Object.assign || function (target, source) { +function shouldUseNative() { + try { + if (!Object.assign) { + return false; + } + + // Detect buggy property enumeration order in older V8 versions. + + // https://bugs.chromium.org/p/v8/issues/detail?id=4118 + var test1 = new String('abc'); // eslint-disable-line + test1[5] = 'de'; + if (Object.getOwnPropertyNames(test1)[0] === '5') { + return false; + } + + // https://bugs.chromium.org/p/v8/issues/detail?id=3056 + var test2 = {}; + for (var i = 0; i < 10; i++) { + test2['_' + String.fromCharCode(i)] = i; + } + var order2 = Object.getOwnPropertyNames(test2).map(function (n) { + return test2[n]; + }); + if (order2.join('') !== '0123456789') { + return false; + } + + // https://bugs.chromium.org/p/v8/issues/detail?id=3056 + var test3 = {}; + 'abcdefghijklmnopqrst'.split('').forEach(function (letter) { + test3[letter] = letter; + }); + if (Object.keys(Object.assign({}, test3)).join('') !== + 'abcdefghijklmnopqrst') { + return false; + } + + return true; + } catch (e) { + // We don't expect any of the above to throw, but better to be safe. + return false; + } +} + +module.exports = shouldUseNative() ? Object.assign : function (target, source) { var from; var to = toObject(target); var symbols; @@ -3041,6 +3087,9 @@ var currentQueue; var queueIndex = -1; function cleanUpNextTick() { + if (!draining || !currentQueue) { + return; + } draining = false; if (currentQueue.length) { queue = currentQueue.concat(queue); @@ -35673,8 +35722,7 @@ return jQuery; (function (global){ /** * @license - * lodash 4.11.2 (Custom Build) <https://lodash.com/> - * Build: `lodash -d -o ./foo/lodash.js` + * lodash <https://lodash.com/> * Copyright jQuery Foundation and other contributors <https://jquery.org/> * Released under MIT license <https://lodash.com/license> * Based on Underscore.js 1.8.3 <http://underscorejs.org/LICENSE> @@ -35686,7 +35734,7 @@ return jQuery; var undefined; /** Used as the semantic version number. */ - var VERSION = '4.11.2'; + var VERSION = '4.12.0'; /** Used as the size to enable large array optimizations. */ var LARGE_ARRAY_SIZE = 200; @@ -36131,30 +36179,6 @@ return jQuery; } /** - * Creates a new array concatenating `array` with `other`. - * - * @private - * @param {Array} array The first array to concatenate. - * @param {Array} other The second array to concatenate. - * @returns {Array} Returns the new concatenated array. - */ - function arrayConcat(array, other) { - var index = -1, - length = array.length, - othIndex = -1, - othLength = other.length, - result = Array(length + othLength); - - while (++index < length) { - result[index] = array[index]; - } - while (++othIndex < othLength) { - result[index++] = other[othIndex]; - } - return result; - } - - /** * A specialized version of `_.forEach` for arrays without support for * iteratee shorthands. * @@ -36581,7 +36605,7 @@ return jQuery; * @private * @param {Object} object The object to query. * @param {Array} props The property names to get values for. - * @returns {Object} Returns the new array of key-value pairs. + * @returns {Object} Returns the key-value pairs. */ function baseToPairs(object, props) { return arrayMap(props, function(key) { @@ -36594,7 +36618,7 @@ return jQuery; * * @private * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new capped function. */ function baseUnary(func) { return function(value) { @@ -36619,6 +36643,18 @@ return jQuery; } /** + * Checks if a cache value for `key` exists. + * + * @private + * @param {Object} cache The cache to query. + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function cacheHas(cache, key) { + return cache.has(key); + } + + /** * Used by `_.trim` and `_.trimStart` to get the index of the first string symbol * that is not found in the character symbols. * @@ -36774,11 +36810,11 @@ return jQuery; } /** - * Converts `map` to an array. + * Converts `map` to its key-value pairs. * * @private * @param {Object} map The map to convert. - * @returns {Array} Returns the converted array. + * @returns {Array} Returns the key-value pairs. */ function mapToArray(map) { var index = -1, @@ -36816,11 +36852,11 @@ return jQuery; } /** - * Converts `set` to an array. + * Converts `set` to an array of its values. * * @private * @param {Object} set The set to convert. - * @returns {Array} Returns the converted array. + * @returns {Array} Returns the values. */ function setToArray(set) { var index = -1, @@ -36833,6 +36869,23 @@ return jQuery; } /** + * Converts `set` to its value-value pairs. + * + * @private + * @param {Object} set The set to convert. + * @returns {Array} Returns the value-value pairs. + */ + function setToPairs(set) { + var index = -1, + result = Array(set.size); + + set.forEach(function(value) { + result[++index] = [value, value]; + }); + return result; + } + + /** * Gets the number of symbols in `string`. * * @private @@ -37085,10 +37138,10 @@ return jQuery; * `floor`, `forEach`, `forEachRight`, `forIn`, `forInRight`, `forOwn`, * `forOwnRight`, `get`, `gt`, `gte`, `has`, `hasIn`, `head`, `identity`, * `includes`, `indexOf`, `inRange`, `invoke`, `isArguments`, `isArray`, - * `isArrayBuffer`, `isArrayLike`, `isArrayLikeObject`, `isBoolean`, `isBuffer`, - * `isDate`, `isElement`, `isEmpty`, `isEqual`, `isEqualWith`, `isError`, - * `isFinite`, `isFunction`, `isInteger`, `isLength`, `isMap`, `isMatch`, - * `isMatchWith`, `isNaN`, `isNative`, `isNil`, `isNull`, `isNumber`, + * `isArrayBuffer`, `isArrayLike`, `isArrayLikeObject`, `isBoolean`, + * `isBuffer`, `isDate`, `isElement`, `isEmpty`, `isEqual`, `isEqualWith`, + * `isError`, `isFinite`, `isFunction`, `isInteger`, `isLength`, `isMap`, + * `isMatch`, `isMatchWith`, `isNaN`, `isNative`, `isNil`, `isNull`, `isNumber`, * `isObject`, `isObjectLike`, `isPlainObject`, `isRegExp`, `isSafeInteger`, * `isSet`, `isString`, `isUndefined`, `isTypedArray`, `isWeakMap`, `isWeakSet`, * `join`, `kebabCase`, `last`, `lastIndexOf`, `lowerCase`, `lowerFirst`, @@ -37097,9 +37150,9 @@ return jQuery; * `pop`, `random`, `reduce`, `reduceRight`, `repeat`, `result`, `round`, * `runInContext`, `sample`, `shift`, `size`, `snakeCase`, `some`, `sortedIndex`, * `sortedIndexBy`, `sortedLastIndex`, `sortedLastIndexBy`, `startCase`, - * `startsWith`, `subtract`, `sum`, `sumBy`, `template`, `times`, `toInteger`, - * `toJSON`, `toLength`, `toLower`, `toNumber`, `toSafeInteger`, `toString`, - * `toUpper`, `trim`, `trimEnd`, `trimStart`, `truncate`, `unescape`, + * `startsWith`, `subtract`, `sum`, `sumBy`, `template`, `times`, `toFinite`, + * `toInteger`, `toJSON`, `toLength`, `toLower`, `toNumber`, `toSafeInteger`, + * `toString`, `toUpper`, `trim`, `trimEnd`, `trimStart`, `truncate`, `unescape`, * `uniqueId`, `upperCase`, `upperFirst`, `value`, and `words` * * @name _ @@ -37359,64 +37412,212 @@ return jQuery; * * @private * @constructor - * @returns {Object} Returns the new hash object. + * @param {Array} [entries] The key-value pairs to cache. */ - function Hash() {} + function Hash(entries) { + var index = -1, + length = entries ? entries.length : 0; + + this.clear(); + while (++index < length) { + var entry = entries[index]; + this.set(entry[0], entry[1]); + } + } + + /** + * Removes all key-value entries from the hash. + * + * @private + * @name clear + * @memberOf Hash + */ + function hashClear() { + this.__data__ = nativeCreate ? nativeCreate(null) : {}; + } /** * Removes `key` and its value from the hash. * * @private + * @name delete + * @memberOf Hash * @param {Object} hash The hash to modify. * @param {string} key The key of the value to remove. * @returns {boolean} Returns `true` if the entry was removed, else `false`. */ - function hashDelete(hash, key) { - return hashHas(hash, key) && delete hash[key]; + function hashDelete(key) { + return this.has(key) && delete this.__data__[key]; } /** * Gets the hash value for `key`. * * @private - * @param {Object} hash The hash to query. + * @name get + * @memberOf Hash * @param {string} key The key of the value to get. * @returns {*} Returns the entry value. */ - function hashGet(hash, key) { + function hashGet(key) { + var data = this.__data__; if (nativeCreate) { - var result = hash[key]; + var result = data[key]; return result === HASH_UNDEFINED ? undefined : result; } - return hasOwnProperty.call(hash, key) ? hash[key] : undefined; + return hasOwnProperty.call(data, key) ? data[key] : undefined; } /** * Checks if a hash value for `key` exists. * * @private - * @param {Object} hash The hash to query. + * @name has + * @memberOf Hash * @param {string} key The key of the entry to check. * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. */ - function hashHas(hash, key) { - return nativeCreate ? hash[key] !== undefined : hasOwnProperty.call(hash, key); + function hashHas(key) { + var data = this.__data__; + return nativeCreate ? data[key] !== undefined : hasOwnProperty.call(data, key); } /** * Sets the hash `key` to `value`. * * @private - * @param {Object} hash The hash to modify. + * @name set + * @memberOf Hash + * @param {string} key The key of the value to set. + * @param {*} value The value to set. + * @returns {Object} Returns the hash instance. + */ + function hashSet(key, value) { + var data = this.__data__; + data[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value; + return this; + } + + // Add methods to `Hash`. + Hash.prototype.clear = hashClear; + Hash.prototype['delete'] = hashDelete; + Hash.prototype.get = hashGet; + Hash.prototype.has = hashHas; + Hash.prototype.set = hashSet; + + /*------------------------------------------------------------------------*/ + + /** + * Creates an list cache object. + * + * @private + * @constructor + * @param {Array} [entries] The key-value pairs to cache. + */ + function ListCache(entries) { + var index = -1, + length = entries ? entries.length : 0; + + this.clear(); + while (++index < length) { + var entry = entries[index]; + this.set(entry[0], entry[1]); + } + } + + /** + * Removes all key-value entries from the list cache. + * + * @private + * @name clear + * @memberOf ListCache + */ + function listCacheClear() { + this.__data__ = []; + } + + /** + * Removes `key` and its value from the list cache. + * + * @private + * @name delete + * @memberOf ListCache + * @param {string} key The key of the value to remove. + * @returns {boolean} Returns `true` if the entry was removed, else `false`. + */ + function listCacheDelete(key) { + var data = this.__data__, + index = assocIndexOf(data, key); + + if (index < 0) { + return false; + } + var lastIndex = data.length - 1; + if (index == lastIndex) { + data.pop(); + } else { + splice.call(data, index, 1); + } + return true; + } + + /** + * Gets the list cache value for `key`. + * + * @private + * @name get + * @memberOf ListCache + * @param {string} key The key of the value to get. + * @returns {*} Returns the entry value. + */ + function listCacheGet(key) { + var data = this.__data__, + index = assocIndexOf(data, key); + + return index < 0 ? undefined : data[index][1]; + } + + /** + * Checks if a list cache value for `key` exists. + * + * @private + * @name has + * @memberOf ListCache + * @param {string} key The key of the entry to check. + * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. + */ + function listCacheHas(key) { + return assocIndexOf(this.__data__, key) > -1; + } + + /** + * Sets the list cache `key` to `value`. + * + * @private + * @name set + * @memberOf ListCache * @param {string} key The key of the value to set. * @param {*} value The value to set. + * @returns {Object} Returns the list cache instance. */ - function hashSet(hash, key, value) { - hash[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value; + function listCacheSet(key, value) { + var data = this.__data__, + index = assocIndexOf(data, key); + + if (index < 0) { + data.push([key, value]); + } else { + data[index][1] = value; + } + return this; } - // Avoid inheriting from `Object.prototype` when possible. - Hash.prototype = nativeCreate ? nativeCreate(null) : objectProto; + // Add methods to `ListCache`. + ListCache.prototype.clear = listCacheClear; + ListCache.prototype['delete'] = listCacheDelete; + ListCache.prototype.get = listCacheGet; + ListCache.prototype.has = listCacheHas; + ListCache.prototype.set = listCacheSet; /*------------------------------------------------------------------------*/ @@ -37425,15 +37626,15 @@ return jQuery; * * @private * @constructor - * @param {Array} [values] The values to cache. + * @param {Array} [entries] The key-value pairs to cache. */ - function MapCache(values) { + function MapCache(entries) { var index = -1, - length = values ? values.length : 0; + length = entries ? entries.length : 0; this.clear(); while (++index < length) { - var entry = values[index]; + var entry = entries[index]; this.set(entry[0], entry[1]); } } @@ -37445,10 +37646,10 @@ return jQuery; * @name clear * @memberOf MapCache */ - function mapClear() { + function mapCacheClear() { this.__data__ = { 'hash': new Hash, - 'map': Map ? new Map : [], + 'map': new (Map || ListCache), 'string': new Hash }; } @@ -37462,12 +37663,8 @@ return jQuery; * @param {string} key The key of the value to remove. * @returns {boolean} Returns `true` if the entry was removed, else `false`. */ - function mapDelete(key) { - var data = this.__data__; - if (isKeyable(key)) { - return hashDelete(typeof key == 'string' ? data.string : data.hash, key); - } - return Map ? data.map['delete'](key) : assocDelete(data.map, key); + function mapCacheDelete(key) { + return getMapData(this, key)['delete'](key); } /** @@ -37479,12 +37676,8 @@ return jQuery; * @param {string} key The key of the value to get. * @returns {*} Returns the entry value. */ - function mapGet(key) { - var data = this.__data__; - if (isKeyable(key)) { - return hashGet(typeof key == 'string' ? data.string : data.hash, key); - } - return Map ? data.map.get(key) : assocGet(data.map, key); + function mapCacheGet(key) { + return getMapData(this, key).get(key); } /** @@ -37496,12 +37689,8 @@ return jQuery; * @param {string} key The key of the entry to check. * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. */ - function mapHas(key) { - var data = this.__data__; - if (isKeyable(key)) { - return hashHas(typeof key == 'string' ? data.string : data.hash, key); - } - return Map ? data.map.has(key) : assocHas(data.map, key); + function mapCacheHas(key) { + return getMapData(this, key).has(key); } /** @@ -37514,30 +37703,23 @@ return jQuery; * @param {*} value The value to set. * @returns {Object} Returns the map cache instance. */ - function mapSet(key, value) { - var data = this.__data__; - if (isKeyable(key)) { - hashSet(typeof key == 'string' ? data.string : data.hash, key, value); - } else if (Map) { - data.map.set(key, value); - } else { - assocSet(data.map, key, value); - } + function mapCacheSet(key, value) { + getMapData(this, key).set(key, value); return this; } // Add methods to `MapCache`. - MapCache.prototype.clear = mapClear; - MapCache.prototype['delete'] = mapDelete; - MapCache.prototype.get = mapGet; - MapCache.prototype.has = mapHas; - MapCache.prototype.set = mapSet; + MapCache.prototype.clear = mapCacheClear; + MapCache.prototype['delete'] = mapCacheDelete; + MapCache.prototype.get = mapCacheGet; + MapCache.prototype.has = mapCacheHas; + MapCache.prototype.set = mapCacheSet; /*------------------------------------------------------------------------*/ /** * - * Creates a set cache object to store unique values. + * Creates an array cache object to store unique values. * * @private * @constructor @@ -37549,52 +37731,41 @@ return jQuery; this.__data__ = new MapCache; while (++index < length) { - this.push(values[index]); + this.add(values[index]); } } /** - * Checks if `value` is in `cache`. + * Adds `value` to the array cache. * * @private - * @param {Object} cache The set cache to search. - * @param {*} value The value to search for. - * @returns {number} Returns `true` if `value` is found, else `false`. + * @name add + * @memberOf SetCache + * @alias push + * @param {*} value The value to cache. + * @returns {Object} Returns the cache instance. */ - function cacheHas(cache, value) { - var map = cache.__data__; - if (isKeyable(value)) { - var data = map.__data__, - hash = typeof value == 'string' ? data.string : data.hash; - - return hash[value] === HASH_UNDEFINED; - } - return map.has(value); + function setCacheAdd(value) { + this.__data__.set(value, HASH_UNDEFINED); + return this; } /** - * Adds `value` to the set cache. + * Checks if `value` is in the array cache. * * @private - * @name push + * @name has * @memberOf SetCache - * @param {*} value The value to cache. + * @param {*} value The value to search for. + * @returns {number} Returns `true` if `value` is found, else `false`. */ - function cachePush(value) { - var map = this.__data__; - if (isKeyable(value)) { - var data = map.__data__, - hash = typeof value == 'string' ? data.string : data.hash; - - hash[value] = HASH_UNDEFINED; - } - else { - map.set(value, HASH_UNDEFINED); - } + function setCacheHas(value) { + return this.__data__.has(value); } // Add methods to `SetCache`. - SetCache.prototype.push = cachePush; + SetCache.prototype.add = SetCache.prototype.push = setCacheAdd; + SetCache.prototype.has = setCacheHas; /*------------------------------------------------------------------------*/ @@ -37603,17 +37774,10 @@ return jQuery; * * @private * @constructor - * @param {Array} [values] The values to cache. + * @param {Array} [entries] The key-value pairs to cache. */ - function Stack(values) { - var index = -1, - length = values ? values.length : 0; - - this.clear(); - while (++index < length) { - var entry = values[index]; - this.set(entry[0], entry[1]); - } + function Stack(entries) { + this.__data__ = new ListCache(entries); } /** @@ -37624,7 +37788,7 @@ return jQuery; * @memberOf Stack */ function stackClear() { - this.__data__ = { 'array': [], 'map': null }; + this.__data__ = new ListCache; } /** @@ -37637,10 +37801,7 @@ return jQuery; * @returns {boolean} Returns `true` if the entry was removed, else `false`. */ function stackDelete(key) { - var data = this.__data__, - array = data.array; - - return array ? assocDelete(array, key) : data.map['delete'](key); + return this.__data__['delete'](key); } /** @@ -37653,10 +37814,7 @@ return jQuery; * @returns {*} Returns the entry value. */ function stackGet(key) { - var data = this.__data__, - array = data.array; - - return array ? assocGet(array, key) : data.map.get(key); + return this.__data__.get(key); } /** @@ -37669,10 +37827,7 @@ return jQuery; * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. */ function stackHas(key) { - var data = this.__data__, - array = data.array; - - return array ? assocHas(array, key) : data.map.has(key); + return this.__data__.has(key); } /** @@ -37686,21 +37841,11 @@ return jQuery; * @returns {Object} Returns the stack cache instance. */ function stackSet(key, value) { - var data = this.__data__, - array = data.array; - - if (array) { - if (array.length < (LARGE_ARRAY_SIZE - 1)) { - assocSet(array, key, value); - } else { - data.array = null; - data.map = new MapCache(array); - } - } - var map = data.map; - if (map) { - map.set(key, value); + var cache = this.__data__; + if (cache instanceof ListCache && cache.__data__.length == LARGE_ARRAY_SIZE) { + cache = this.__data__ = new MapCache(cache.__data__); } + cache.set(key, value); return this; } @@ -37714,90 +37859,6 @@ return jQuery; /*------------------------------------------------------------------------*/ /** - * Removes `key` and its value from the associative array. - * - * @private - * @param {Array} array The array to modify. - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. - */ - function assocDelete(array, key) { - var index = assocIndexOf(array, key); - if (index < 0) { - return false; - } - var lastIndex = array.length - 1; - if (index == lastIndex) { - array.pop(); - } else { - splice.call(array, index, 1); - } - return true; - } - - /** - * Gets the associative array value for `key`. - * - * @private - * @param {Array} array The array to query. - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. - */ - function assocGet(array, key) { - var index = assocIndexOf(array, key); - return index < 0 ? undefined : array[index][1]; - } - - /** - * Checks if an associative array value for `key` exists. - * - * @private - * @param {Array} array The array to query. - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function assocHas(array, key) { - return assocIndexOf(array, key) > -1; - } - - /** - * Gets the index at which the `key` is found in `array` of key-value pairs. - * - * @private - * @param {Array} array The array to search. - * @param {*} key The key to search for. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function assocIndexOf(array, key) { - var length = array.length; - while (length--) { - if (eq(array[length][0], key)) { - return length; - } - } - return -1; - } - - /** - * Sets the associative array `key` to `value`. - * - * @private - * @param {Array} array The array to modify. - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - */ - function assocSet(array, key, value) { - var index = assocIndexOf(array, key); - if (index < 0) { - array.push([key, value]); - } else { - array[index][1] = value; - } - } - - /*------------------------------------------------------------------------*/ - - /** * Used by `_.defaults` to customize its `_.assignIn` use. * * @private @@ -37850,6 +37911,24 @@ return jQuery; } /** + * Gets the index at which the `key` is found in `array` of key-value pairs. + * + * @private + * @param {Array} array The array to search. + * @param {*} key The key to search for. + * @returns {number} Returns the index of the matched value, else `-1`. + */ + function assocIndexOf(array, key) { + var length = array.length; + while (length--) { + if (eq(array[length][0], key)) { + return length; + } + } + return -1; + } + + /** * Aggregates elements of `collection` on `accumulator` with keys transformed * by `iteratee` and values set by `setter`. * @@ -37886,7 +37965,7 @@ return jQuery; * @private * @param {Object} object The object to iterate over. * @param {string[]} paths The property paths of elements to pick. - * @returns {Array} Returns the new array of picked elements. + * @returns {Array} Returns the picked elements. */ function baseAt(object, paths) { var index = -1, @@ -38001,7 +38080,7 @@ return jQuery; * * @private * @param {Object} source The object of property predicates to conform to. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. */ function baseConforms(source) { var props = keys(source), @@ -38314,7 +38393,7 @@ return jQuery; * @private * @param {Object} object The object to inspect. * @param {Array} props The property names to filter. - * @returns {Array} Returns the new array of filtered property names. + * @returns {Array} Returns the function names. */ function baseFunctions(object, props) { return arrayFilter(props, function(key) { @@ -38355,9 +38434,7 @@ return jQuery; */ function baseGetAllKeys(object, keysFunc, symbolsFunc) { var result = keysFunc(object); - return isArray(object) - ? result - : arrayPush(result, symbolsFunc(object)); + return isArray(object) ? result : arrayPush(result, symbolsFunc(object)); } /** @@ -38749,7 +38826,7 @@ return jQuery; * * @private * @param {Object} source The object of property values to match. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. */ function baseMatches(source) { var matchData = getMatchData(source); @@ -38767,7 +38844,7 @@ return jQuery; * @private * @param {string} path The path of the property to get. * @param {*} srcValue The value to match. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. */ function baseMatchesProperty(path, srcValue) { if (isKey(path) && isStrictComparable(srcValue)) { @@ -38982,7 +39059,7 @@ return jQuery; * * @private * @param {string} key The key of the property to get. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new accessor function. */ function baseProperty(key) { return function(object) { @@ -38995,7 +39072,7 @@ return jQuery; * * @private * @param {Array|string} path The path of the property to get. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new accessor function. */ function basePropertyDeep(path) { return function(object) { @@ -39096,7 +39173,7 @@ return jQuery; * @param {number} end The end of the range. * @param {number} step The value to increment or decrement by. * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Array} Returns the new array of numbers. + * @returns {Array} Returns the range of numbers. */ function baseRange(start, end, step, fromRight) { var index = -1, @@ -39810,7 +39887,7 @@ return jQuery; * placeholders, and provided arguments into a single array of arguments. * * @private - * @param {Array|Object} args The provided arguments. + * @param {Array} args The provided arguments. * @param {Array} partials The arguments to prepend to those provided. * @param {Array} holders The `partials` placeholder indexes. * @params {boolean} [isCurried] Specify composing for a curried function. @@ -39845,7 +39922,7 @@ return jQuery; * is tailored for `_.partialRight`. * * @private - * @param {Array|Object} args The provided arguments. + * @param {Array} args The provided arguments. * @param {Array} partials The arguments to append to those provided. * @param {Array} holders The `partials` placeholder indexes. * @params {boolean} [isCurried] Specify composing for a curried function. @@ -39967,7 +40044,7 @@ return jQuery; customizer = length > 1 ? sources[length - 1] : undefined, guard = length > 2 ? sources[2] : undefined; - customizer = typeof customizer == 'function' + customizer = (assigner.length > 3 && typeof customizer == 'function') ? (length--, customizer) : undefined; @@ -40066,7 +40143,7 @@ return jQuery; * * @private * @param {string} methodName The name of the `String` case method to use. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new case function. */ function createCaseFirst(methodName) { return function(string) { @@ -40151,7 +40228,7 @@ return jQuery; var length = arguments.length, args = Array(length), index = length, - placeholder = getPlaceholder(wrapper); + placeholder = getHolder(wrapper); while (index--) { args[index] = arguments[index]; @@ -40266,14 +40343,14 @@ return jQuery; function wrapper() { var length = arguments.length, - index = length, - args = Array(length); + args = Array(length), + index = length; while (index--) { args[index] = arguments[index]; } if (isCurried) { - var placeholder = getPlaceholder(wrapper), + var placeholder = getHolder(wrapper), holdersCount = countHolders(args, placeholder); } if (partials) { @@ -40362,7 +40439,7 @@ return jQuery; * * @private * @param {Function} arrayFunc The function to iterate over iteratees. - * @returns {Function} Returns the new invoker function. + * @returns {Function} Returns the new over function. */ function createOver(arrayFunc) { return rest(function(iteratees) { @@ -40561,6 +40638,26 @@ return jQuery; }; /** + * Creates a `_.toPairs` or `_.toPairsIn` function. + * + * @private + * @param {Function} keysFunc The function to get the keys of a given object. + * @returns {Function} Returns the new pairs function. + */ + function createToPairs(keysFunc) { + return function(object) { + var tag = getTag(object); + if (tag == mapTag) { + return mapToArray(object); + } + if (tag == setTag) { + return setToPairs(object); + } + return baseToPairs(object, keysFunc(object)); + }; + } + + /** * Creates a function that either curries or invokes `func` with optional * `this` binding and partially applied arguments. * @@ -40577,6 +40674,7 @@ return jQuery; * 64 - `_.partialRight` * 128 - `_.rearg` * 256 - `_.ary` + * 512 - `_.flip` * @param {*} [thisArg] The `this` binding of `func`. * @param {Array} [partials] The arguments to be partially applied. * @param {Array} [holders] The `partials` placeholder indexes. @@ -40655,9 +40753,7 @@ return jQuery; * @returns {boolean} Returns `true` if the arrays are equivalent, else `false`. */ function equalArrays(array, other, equalFunc, customizer, bitmask, stack) { - var index = -1, - isPartial = bitmask & PARTIAL_COMPARE_FLAG, - isUnordered = bitmask & UNORDERED_COMPARE_FLAG, + var isPartial = bitmask & PARTIAL_COMPARE_FLAG, arrLength = array.length, othLength = other.length; @@ -40669,7 +40765,10 @@ return jQuery; if (stacked) { return stacked == other; } - var result = true; + var index = -1, + result = true, + seen = (bitmask & UNORDERED_COMPARE_FLAG) ? new SetCache : undefined; + stack.set(array, other); // Ignore non-index properties. @@ -40690,10 +40789,12 @@ return jQuery; break; } // Recursively compare arrays (susceptible to call stack limits). - if (isUnordered) { - if (!arraySome(other, function(othValue) { - return arrValue === othValue || - equalFunc(arrValue, othValue, customizer, bitmask, stack); + if (seen) { + if (!arraySome(other, function(othValue, othIndex) { + if (!seen.has(othIndex) && + (arrValue === othValue || equalFunc(arrValue, othValue, customizer, bitmask, stack))) { + return seen.add(othIndex); + } })) { result = false; break; @@ -40928,6 +41029,18 @@ return jQuery; } /** + * Gets the argument placeholder value for `func`. + * + * @private + * @param {Function} func The function to inspect. + * @returns {*} Returns the placeholder value. + */ + function getHolder(func) { + var object = hasOwnProperty.call(lodash, 'placeholder') ? lodash : func; + return object.placeholder; + } + + /** * Gets the appropriate "iteratee" function. If `_.iteratee` is customized, * this function returns the custom method, otherwise it returns `baseIteratee`. * If arguments are provided, the chosen function is invoked with them and @@ -40958,6 +41071,21 @@ return jQuery; var getLength = baseProperty('length'); /** + * Gets the data for `map`. + * + * @private + * @param {Object} map The map to query. + * @param {string} key The reference key. + * @returns {*} Returns the map data. + */ + function getMapData(map, key) { + var data = map.__data__; + return isKeyable(key) + ? data[typeof key == 'string' ? 'string' : 'hash'] + : data.map; + } + + /** * Gets the property names, values, and compare flags of `object`. * * @private @@ -40988,18 +41116,6 @@ return jQuery; } /** - * Gets the argument placeholder value for `func`. - * - * @private - * @param {Function} func The function to inspect. - * @returns {*} Returns the placeholder value. - */ - function getPlaceholder(func) { - var object = hasOwnProperty.call(lodash, 'placeholder') ? lodash : func; - return object.placeholder; - } - - /** * Gets the `[[Prototype]]` of `value`. * * @private @@ -41248,7 +41364,7 @@ return jQuery; * @returns {boolean} Returns `true` if `value` is flattenable, else `false`. */ function isFlattenable(value) { - return isArrayLikeObject(value) && (isArray(value) || isArguments(value)); + return isArray(value) || isArguments(value); } /** @@ -41392,7 +41508,7 @@ return jQuery; * @private * @param {string} key The key of the property to get. * @param {*} srcValue The value to match. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. */ function matchesStrictComparable(key, srcValue) { return function(object) { @@ -41644,7 +41760,7 @@ return jQuery; * @param {Array} array The array to process. * @param {number} [size=1] The length of each chunk * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the new array containing chunks. + * @returns {Array} Returns the new array of chunks. * @example * * _.chunk(['a', 'b', 'c', 'd'], 2); @@ -41727,16 +41843,16 @@ return jQuery; */ function concat() { var length = arguments.length, - array = castArray(arguments[0]); + args = Array(length ? length - 1 : 0), + array = arguments[0], + index = length; - if (length < 2) { - return length ? copyArray(array) : []; - } - var args = Array(length - 1); - while (length--) { - args[length - 1] = arguments[length]; + while (index--) { + args[index - 1] = arguments[index]; } - return arrayConcat(array, baseFlatten(args, 1)); + return length + ? arrayPush(isArray(array) ? copyArray(array) : [array], baseFlatten(args, 1)) + : []; } /** @@ -42455,8 +42571,8 @@ return jQuery; } /** - * Gets the nth element of `array`. If `n` is negative, the nth element - * from the end is returned. + * Gets the element at `n` index of `array`. If `n` is negative, the nth + * element from the end is returned. * * @static * @memberOf _ @@ -43336,7 +43452,7 @@ return jQuery; * @memberOf _ * @since 0.1.0 * @category Array - * @param {Array} array The array to filter. + * @param {Array} array The array to inspect. * @param {...*} [values] The values to exclude. * @returns {Array} Returns the new array of filtered values. * @see _.difference, _.xor @@ -43362,7 +43478,7 @@ return jQuery; * @since 2.4.0 * @category Array * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of values. + * @returns {Array} Returns the new array of filtered values. * @see _.difference, _.without * @example * @@ -43386,7 +43502,7 @@ return jQuery; * @param {...Array} [arrays] The arrays to inspect. * @param {Array|Function|Object|string} [iteratee=_.identity] * The iteratee invoked per element. - * @returns {Array} Returns the new array of values. + * @returns {Array} Returns the new array of filtered values. * @example * * _.xorBy([2.1, 1.2], [4.3, 2.4], Math.floor); @@ -43415,7 +43531,7 @@ return jQuery; * @category Array * @param {...Array} [arrays] The arrays to inspect. * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of values. + * @returns {Array} Returns the new array of filtered values. * @example * * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; @@ -44163,9 +44279,8 @@ return jQuery; * // => Logs 'a' then 'b' (iteration order is not guaranteed). */ function forEach(collection, iteratee) { - return (typeof iteratee == 'function' && isArray(collection)) - ? arrayEach(collection, iteratee) - : baseEach(collection, getIteratee(iteratee)); + var func = isArray(collection) ? arrayEach : baseEach; + return func(collection, getIteratee(iteratee, 3)); } /** @@ -44189,9 +44304,8 @@ return jQuery; * // => Logs `2` then `1`. */ function forEachRight(collection, iteratee) { - return (typeof iteratee == 'function' && isArray(collection)) - ? arrayEachRight(collection, iteratee) - : baseEachRight(collection, getIteratee(iteratee)); + var func = isArray(collection) ? arrayEachRight : baseEachRight; + return func(collection, getIteratee(iteratee, 3)); } /** @@ -44872,7 +44986,7 @@ return jQuery; * @param {Function} func The function to cap arguments for. * @param {number} [n=func.length] The arity cap. * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new capped function. * @example * * _.map(['6', '8', '10'], _.ary(parseInt, 1)); @@ -44956,7 +45070,7 @@ return jQuery; var bind = rest(function(func, thisArg, partials) { var bitmask = BIND_FLAG; if (partials.length) { - var holders = replaceHolders(partials, getPlaceholder(bind)); + var holders = replaceHolders(partials, getHolder(bind)); bitmask |= PARTIAL_FLAG; } return createWrapper(func, bitmask, thisArg, partials, holders); @@ -45010,7 +45124,7 @@ return jQuery; var bindKey = rest(function(object, key, partials) { var bitmask = BIND_FLAG | BIND_KEY_FLAG; if (partials.length) { - var holders = replaceHolders(partials, getPlaceholder(bindKey)); + var holders = replaceHolders(partials, getHolder(bindKey)); bitmask |= PARTIAL_FLAG; } return createWrapper(key, bitmask, object, partials, holders); @@ -45336,7 +45450,7 @@ return jQuery; * @since 4.0.0 * @category Function * @param {Function} func The function to flip arguments for. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new flipped function. * @example * * var flipped = _.flip(function() { @@ -45369,7 +45483,7 @@ return jQuery; * @category Function * @param {Function} func The function to have its output memoized. * @param {Function} [resolver] The function to resolve the cache key. - * @returns {Function} Returns the new memoizing function. + * @returns {Function} Returns the new memoized function. * @example * * var object = { 'a': 1, 'b': 2 }; @@ -45427,7 +45541,7 @@ return jQuery; * @since 3.0.0 * @category Function * @param {Function} predicate The predicate to negate. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new negated function. * @example * * function isEven(n) { @@ -45551,7 +45665,7 @@ return jQuery; * // => 'hi fred' */ var partial = rest(function(func, partials) { - var holders = replaceHolders(partials, getPlaceholder(partial)); + var holders = replaceHolders(partials, getHolder(partial)); return createWrapper(func, PARTIAL_FLAG, undefined, partials, holders); }); @@ -45588,7 +45702,7 @@ return jQuery; * // => 'hello fred' */ var partialRight = rest(function(func, partials) { - var holders = replaceHolders(partials, getPlaceholder(partialRight)); + var holders = replaceHolders(partials, getHolder(partialRight)); return createWrapper(func, PARTIAL_RIGHT_FLAG, undefined, partials, holders); }); @@ -45790,7 +45904,7 @@ return jQuery; * @since 4.0.0 * @category Function * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new capped function. * @example * * _.map(['6', '8', '10'], _.unary(parseInt)); @@ -46466,14 +46580,14 @@ return jQuery; * _.isFinite(3); * // => true * - * _.isFinite(Number.MAX_VALUE); - * // => true - * - * _.isFinite(3.14); + * _.isFinite(Number.MIN_VALUE); * // => true * * _.isFinite(Infinity); * // => false + * + * _.isFinite('3'); + * // => false */ function isFinite(value) { return typeof value == 'number' && nativeIsFinite(value); @@ -47195,6 +47309,41 @@ return jQuery; } /** + * Converts `value` to a finite number. + * + * @static + * @memberOf _ + * @since 4.12.0 + * @category Lang + * @param {*} value The value to convert. + * @returns {number} Returns the converted number. + * @example + * + * _.toFinite(3.2); + * // => 3.2 + * + * _.toFinite(Number.MIN_VALUE); + * // => 5e-324 + * + * _.toFinite(Infinity); + * // => 1.7976931348623157e+308 + * + * _.toFinite('3.2'); + * // => 3.2 + */ + function toFinite(value) { + if (!value) { + return value === 0 ? value : 0; + } + value = toNumber(value); + if (value === INFINITY || value === -INFINITY) { + var sign = (value < 0 ? -1 : 1); + return sign * MAX_INTEGER; + } + return value === value ? value : 0; + } + + /** * Converts `value` to an integer. * * **Note:** This function is loosely based on @@ -47208,7 +47357,7 @@ return jQuery; * @returns {number} Returns the converted integer. * @example * - * _.toInteger(3); + * _.toInteger(3.2); * // => 3 * * _.toInteger(Number.MIN_VALUE); @@ -47217,20 +47366,14 @@ return jQuery; * _.toInteger(Infinity); * // => 1.7976931348623157e+308 * - * _.toInteger('3'); + * _.toInteger('3.2'); * // => 3 */ function toInteger(value) { - if (!value) { - return value === 0 ? value : 0; - } - value = toNumber(value); - if (value === INFINITY || value === -INFINITY) { - var sign = (value < 0 ? -1 : 1); - return sign * MAX_INTEGER; - } - var remainder = value % 1; - return value === value ? (remainder ? value - remainder : value) : 0; + var result = toFinite(value), + remainder = result % 1; + + return result === result ? (remainder ? result - remainder : result) : 0; } /** @@ -47248,7 +47391,7 @@ return jQuery; * @returns {number} Returns the converted integer. * @example * - * _.toLength(3); + * _.toLength(3.2); * // => 3 * * _.toLength(Number.MIN_VALUE); @@ -47257,7 +47400,7 @@ return jQuery; * _.toLength(Infinity); * // => 4294967295 * - * _.toLength('3'); + * _.toLength('3.2'); * // => 3 */ function toLength(value) { @@ -47275,8 +47418,8 @@ return jQuery; * @returns {number} Returns the number. * @example * - * _.toNumber(3); - * // => 3 + * _.toNumber(3.2); + * // => 3.2 * * _.toNumber(Number.MIN_VALUE); * // => 5e-324 @@ -47284,8 +47427,8 @@ return jQuery; * _.toNumber(Infinity); * // => Infinity * - * _.toNumber('3'); - * // => 3 + * _.toNumber('3.2'); + * // => 3.2 */ function toNumber(value) { if (typeof value == 'number') { @@ -47348,7 +47491,7 @@ return jQuery; * @returns {number} Returns the converted integer. * @example * - * _.toSafeInteger(3); + * _.toSafeInteger(3.2); * // => 3 * * _.toSafeInteger(Number.MIN_VALUE); @@ -47357,7 +47500,7 @@ return jQuery; * _.toSafeInteger(Infinity); * // => 9007199254740991 * - * _.toSafeInteger('3'); + * _.toSafeInteger('3.2'); * // => 3 */ function toSafeInteger(value) { @@ -47550,7 +47693,7 @@ return jQuery; * @category Object * @param {Object} object The object to iterate over. * @param {...(string|string[])} [paths] The property paths of elements to pick. - * @returns {Array} Returns the new array of picked elements. + * @returns {Array} Returns the picked values. * @example * * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; @@ -47766,7 +47909,7 @@ return jQuery; function forIn(object, iteratee) { return object == null ? object - : baseFor(object, getIteratee(iteratee), keysIn); + : baseFor(object, getIteratee(iteratee, 3), keysIn); } /** @@ -47798,7 +47941,7 @@ return jQuery; function forInRight(object, iteratee) { return object == null ? object - : baseForRight(object, getIteratee(iteratee), keysIn); + : baseForRight(object, getIteratee(iteratee, 3), keysIn); } /** @@ -47830,7 +47973,7 @@ return jQuery; * // => Logs 'a' then 'b' (iteration order is not guaranteed). */ function forOwn(object, iteratee) { - return object && baseForOwn(object, getIteratee(iteratee)); + return object && baseForOwn(object, getIteratee(iteratee, 3)); } /** @@ -47860,7 +48003,7 @@ return jQuery; * // => Logs 'b' then 'a' assuming `_.forOwn` logs 'a' then 'b'. */ function forOwnRight(object, iteratee) { - return object && baseForOwnRight(object, getIteratee(iteratee)); + return object && baseForOwnRight(object, getIteratee(iteratee, 3)); } /** @@ -47872,7 +48015,7 @@ return jQuery; * @memberOf _ * @category Object * @param {Object} object The object to inspect. - * @returns {Array} Returns the new array of property names. + * @returns {Array} Returns the function names. * @see _.functionsIn * @example * @@ -47899,7 +48042,7 @@ return jQuery; * @since 4.0.0 * @category Object * @param {Object} object The object to inspect. - * @returns {Array} Returns the new array of property names. + * @returns {Array} Returns the function names. * @see _.functions * @example * @@ -48252,7 +48395,7 @@ return jQuery; * inherited enumerable string keyed properties of source objects into the * destination object. Source properties that resolve to `undefined` are * skipped if a destination value exists. Array and plain object properties - * are merged recursively.Other objects and value types are overridden by + * are merged recursively. Other objects and value types are overridden by * assignment. Source objects are applied from left to right. Subsequent * sources overwrite property assignments of previous sources. * @@ -48537,7 +48680,8 @@ return jQuery; /** * Creates an array of own enumerable string keyed-value pairs for `object` - * which can be consumed by `_.fromPairs`. + * which can be consumed by `_.fromPairs`. If `object` is a map or set, its + * entries are returned. * * @static * @memberOf _ @@ -48545,7 +48689,7 @@ return jQuery; * @alias entries * @category Object * @param {Object} object The object to query. - * @returns {Array} Returns the new array of key-value pairs. + * @returns {Array} Returns the key-value pairs. * @example * * function Foo() { @@ -48558,13 +48702,12 @@ return jQuery; * _.toPairs(new Foo); * // => [['a', 1], ['b', 2]] (iteration order is not guaranteed) */ - function toPairs(object) { - return baseToPairs(object, keys(object)); - } + var toPairs = createToPairs(keys); /** * Creates an array of own and inherited enumerable string keyed-value pairs - * for `object` which can be consumed by `_.fromPairs`. + * for `object` which can be consumed by `_.fromPairs`. If `object` is a map + * or set, its entries are returned. * * @static * @memberOf _ @@ -48572,7 +48715,7 @@ return jQuery; * @alias entriesIn * @category Object * @param {Object} object The object to query. - * @returns {Array} Returns the new array of key-value pairs. + * @returns {Array} Returns the key-value pairs. * @example * * function Foo() { @@ -48583,11 +48726,9 @@ return jQuery; * Foo.prototype.c = 3; * * _.toPairsIn(new Foo); - * // => [['a', 1], ['b', 2], ['c', 1]] (iteration order is not guaranteed) + * // => [['a', 1], ['b', 2], ['c', 3]] (iteration order is not guaranteed) */ - function toPairsIn(object) { - return baseToPairs(object, keysIn(object)); - } + var toPairsIn = createToPairs(keysIn); /** * An alternative to `_.reduce`; this method transforms `object` to a new @@ -49417,7 +49558,7 @@ return jQuery; * @param {string} [string=''] The string to split. * @param {RegExp|string} separator The separator pattern to split by. * @param {number} [limit] The length to truncate results to. - * @returns {Array} Returns the new array of string segments. + * @returns {Array} Returns the string segments. * @example * * _.split('a-b-c', '-', 2); @@ -49562,12 +49703,6 @@ return jQuery; * compiled({ 'user': 'pebbles' }); * // => 'hello pebbles!' * - * // Use custom template delimiters. - * _.templateSettings.interpolate = /{{([\s\S]+?)}}/g; - * var compiled = _.template('hello {{ user }}!'); - * compiled({ 'user': 'mustache' }); - * // => 'hello mustache!' - * * // Use backslashes to treat delimiters as plain text. * var compiled = _.template('<%= "\\<%- value %\\>" %>'); * compiled({ 'value': 'ignored' }); @@ -49593,9 +49728,15 @@ return jQuery; * // return __p; * // } * + * // Use custom template delimiters. + * _.templateSettings.interpolate = /{{([\s\S]+?)}}/g; + * var compiled = _.template('hello {{ user }}!'); + * compiled({ 'user': 'mustache' }); + * // => 'hello mustache!' + * * // Use the `source` property to inline compiled templates for meaningful * // line numbers in error messages and stack traces. - * fs.writeFileSync(path.join(cwd, 'jst.js'), '\ + * fs.writeFileSync(path.join(process.cwd(), 'jst.js'), '\ * var JST = {\ * "main": ' + _.template(mainText).source + '\ * };\ @@ -50131,7 +50272,7 @@ return jQuery; * @since 4.0.0 * @category Util * @param {Array} pairs The predicate-function pairs. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new composite function. * @example * * var func = _.cond([ @@ -50181,7 +50322,7 @@ return jQuery; * @since 4.0.0 * @category Util * @param {Object} source The object of property predicates to conform to. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. * @example * * var users = [ @@ -50204,7 +50345,7 @@ return jQuery; * @since 2.4.0 * @category Util * @param {*} value The value to return from the new function. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new constant function. * @example * * var object = { 'user': 'fred' }; @@ -50229,7 +50370,7 @@ return jQuery; * @since 3.0.0 * @category Util * @param {...(Function|Function[])} [funcs] Functions to invoke. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new composite function. * @see _.flowRight * @example * @@ -50252,7 +50393,7 @@ return jQuery; * @memberOf _ * @category Util * @param {...(Function|Function[])} [funcs] Functions to invoke. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new composite function. * @see _.flow * @example * @@ -50345,7 +50486,7 @@ return jQuery; * @since 3.0.0 * @category Util * @param {Object} source The object of property values to match. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. * @example * * var users = [ @@ -50373,7 +50514,7 @@ return jQuery; * @category Util * @param {Array|string} path The path of the property to get. * @param {*} srcValue The value to match. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new spec function. * @example * * var users = [ @@ -50398,7 +50539,7 @@ return jQuery; * @category Util * @param {Array|string} path The path of the method to invoke. * @param {...*} [args] The arguments to invoke the method with. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new invoker function. * @example * * var objects = [ @@ -50429,7 +50570,7 @@ return jQuery; * @category Util * @param {Object} object The object to query. * @param {...*} [args] The arguments to invoke the method with. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new invoker function. * @example * * var array = _.times(3, _.constant), @@ -50559,7 +50700,7 @@ return jQuery; } /** - * Creates a function that returns its nth argument. If `n` is negative, + * Creates a function that gets the argument at `n` index. If `n` is negative, * the nth argument from the end is returned. * * @static @@ -50567,7 +50708,7 @@ return jQuery; * @since 4.0.0 * @category Util * @param {number} [n=0] The index of the argument to return. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new pass-thru function. * @example * * var func = _.nthArg(1); @@ -50665,7 +50806,7 @@ return jQuery; * @since 2.4.0 * @category Util * @param {Array|string} path The path of the property to get. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new accessor function. * @example * * var objects = [ @@ -50692,7 +50833,7 @@ return jQuery; * @since 3.0.0 * @category Util * @param {Object} object The object to query. - * @returns {Function} Returns the new function. + * @returns {Function} Returns the new accessor function. * @example * * var array = [0, 1, 2], @@ -50726,7 +50867,7 @@ return jQuery; * @param {number} [start=0] The start of the range. * @param {number} end The end of the range. * @param {number} [step=1] The value to increment or decrement by. - * @returns {Array} Returns the new array of numbers. + * @returns {Array} Returns the range of numbers. * @see _.inRange, _.rangeRight * @example * @@ -50764,7 +50905,7 @@ return jQuery; * @param {number} [start=0] The start of the range. * @param {number} end The end of the range. * @param {number} [step=1] The value to increment or decrement by. - * @returns {Array} Returns the new array of numbers. + * @returns {Array} Returns the range of numbers. * @see _.inRange, _.range * @example * @@ -51525,6 +51666,7 @@ return jQuery; lodash.sumBy = sumBy; lodash.template = template; lodash.times = times; + lodash.toFinite = toFinite; lodash.toInteger = toInteger; lodash.toLength = toLength; lodash.toLower = toLower; diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index caa84ff7..4451f1da 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -1,5 +1,6 @@ from six.moves import http_cookies as Cookie import re +import string from email.utils import parsedate_tz, formatdate, mktime_tz from .. import odict @@ -27,7 +28,6 @@ variants. Serialization follows RFC6265. # TODO: Disallow LHS-only Cookie values - def _read_until(s, start, term): """ Read until one of the characters in term is reached. @@ -203,25 +203,26 @@ def refresh_set_cookie_header(c, delta): Returns: A refreshed Set-Cookie string """ - try: - c = Cookie.SimpleCookie(str(c)) - except Cookie.CookieError: + + name, value, attrs = parse_set_cookie_header(c) + if not name or not value: raise ValueError("Invalid Cookie") - for i in c.values(): - if "expires" in i: - d = parsedate_tz(i["expires"]) - if d: - d = mktime_tz(d) + delta - i["expires"] = formatdate(d) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del i["expires"] - ret = c.output(header="").strip() + + if "expires" in attrs: + e = parsedate_tz(attrs["expires"][-1]) + if e: + f = mktime_tz(e) + delta + attrs["expires"] = [formatdate(f)] + else: + # This can happen when the expires tag is invalid. + # reddit.com sends a an expires tag like this: "Thu, 31 Dec + # 2037 23:59:59 GMT", which is valid RFC 1123, but not + # strictly correct according to the cookie spec. Browsers + # appear to parse this tolerantly - maybe we should too. + # For now, we just ignore this. + del attrs["expires"] + + ret = format_set_cookie_header(name, value, attrs) if not ret: raise ValueError("Invalid Cookie") return ret diff --git a/netlib/http/request.py b/netlib/http/request.py index 6406a980..a42150ff 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -162,7 +162,7 @@ class Request(Message): def path(self): """ HTTP request path, e.g. "/index.html". - Guaranteed to start with a slash. + Guaranteed to start with a slash, except for OPTIONS requests, which may just be "*". """ if self.data.path is None: return None @@ -343,14 +343,6 @@ class Request(Message): # Legacy - def get_cookies(self): # pragma: no cover - warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) - return self.cookies - - def set_cookies(self, odict): # pragma: no cover - warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) - self.cookies = odict - def get_query(self): # pragma: no cover warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning) return self.query or ODict([]) diff --git a/netlib/http/response.py b/netlib/http/response.py index efd7f60a..2f06149e 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -127,13 +127,3 @@ class Response(Message): c.append(refreshed) if c: self.headers.set_all("set-cookie", c) - - # Legacy - - def get_cookies(self): # pragma: no cover - warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning) - return self.cookies - - def set_cookies(self, odict): # pragma: no cover - warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning) - self.cookies = odict diff --git a/netlib/odict.py b/netlib/odict.py index 461192f7..8a638dab 100644 --- a/netlib/odict.py +++ b/netlib/odict.py @@ -1,5 +1,6 @@ from __future__ import (absolute_import, print_function, division) import copy + import six from .utils import Serializable, safe_subn @@ -27,27 +28,24 @@ class ODict(Serializable): def __iter__(self): return self.lst.__iter__() - def __getitem__(self, k): + def __getitem__(self, key): """ Returns a list of values matching key. """ - ret = [] - k = self._kconv(k) - for i in self.lst: - if self._kconv(i[0]) == k: - ret.append(i[1]) - return ret - def keys(self): - return list(set([self._kconv(i[0]) for i in self.lst])) + key = self._kconv(key) + return [ + v + for k, v in self.lst + if self._kconv(k) == key + ] - def _filter_lst(self, k, lst): - k = self._kconv(k) - new = [] - for i in lst: - if self._kconv(i[0]) != k: - new.append(i) - return new + def keys(self): + return list( + set( + self._kconv(k) for k, _ in self.lst + ) + ) def __len__(self): """ @@ -81,14 +79,19 @@ class ODict(Serializable): """ Delete all items matching k. """ - self.lst = self._filter_lst(k, self.lst) - - def __contains__(self, k): k = self._kconv(k) - for i in self.lst: - if self._kconv(i[0]) == k: - return True - return False + self.lst = [ + i + for i in self.lst + if self._kconv(i[0]) != k + ] + + def __contains__(self, key): + key = self._kconv(key) + return any( + self._kconv(k) == key + for k, _ in self.lst + ) def add(self, key, value, prepend=False): if prepend: @@ -127,40 +130,24 @@ class ODict(Serializable): def __repr__(self): return repr(self.lst) - def in_any(self, key, value, caseless=False): - """ - Do any of the values matching key contain value? - - If caseless is true, value comparison is case-insensitive. - """ - if caseless: - value = value.lower() - for i in self[key]: - if caseless: - i = i.lower() - if value in i: - return True - return False - def replace(self, pattern, repl, *args, **kwargs): """ Replaces a regular expression pattern with repl in both keys and - values. Encoded content will be decoded before replacement, and - re-encoded afterwards. + values. Returns the number of replacements made. """ - nlst, count = [], 0 - for i in self.lst: - k, c = safe_subn(pattern, repl, i[0], *args, **kwargs) + new, count = [], 0 + for k, v in self.lst: + k, c = safe_subn(pattern, repl, k, *args, **kwargs) count += c - v, c = safe_subn(pattern, repl, i[1], *args, **kwargs) + v, c = safe_subn(pattern, repl, v, *args, **kwargs) count += c - nlst.append([k, v]) - self.lst = nlst + new.append([k, v]) + self.lst = new return count - # Implement the StateObject protocol from mitmproxy + # Implement Serializable def get_state(self): return [tuple(i) for i in self.lst] diff --git a/netlib/utils.py b/netlib/utils.py index dda76808..be2701a0 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -330,6 +330,8 @@ def unparse_url(scheme, host, port, path=""): Args: All args must be str. """ + if path == "*": + path = "" return "%s://%s%s" % (scheme, hostport(scheme, host, port), path) @@ -429,3 +431,31 @@ def safe_subn(pattern, repl, target, *args, **kwargs): need a better solution that is aware of the actual content ecoding. """ return re.subn(str(pattern), str(repl), target, *args, **kwargs) + + +def bytes_to_escaped_str(data): + """ + Take bytes and return a safe string that can be displayed to the user. + """ + # TODO: We may want to support multi-byte characters without escaping them. + # One way to do would be calling .decode("utf8", "backslashreplace") first + # and then escaping UTF8 control chars (see clean_bin). + + if not isinstance(data, bytes): + raise ValueError("data must be bytes") + return repr(data).lstrip("b")[1:-1] + + +def escaped_str_to_bytes(data): + """ + Take an escaped string and return the unescaped bytes equivalent. + """ + if not isinstance(data, str): + raise ValueError("data must be str") + + if six.PY2: + return data.decode("string-escape") + + # This one is difficult - we use an undocumented Python API here + # as per http://stackoverflow.com/a/23151714/934719 + return codecs.escape_decode(data)[0] diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py index ea7c870e..09443a95 100644 --- a/pathod/language/websockets.py +++ b/pathod/language/websockets.py @@ -1,4 +1,6 @@ import os +import random +import string import netlib.websockets import pyparsing as pp from . import base, generators, actions, message @@ -175,8 +177,10 @@ class WebsocketFrame(message.Message): Mask(True) ) if not self.knone and self.mask and self.mask.value and not self.key: + allowed_chars = string.ascii_letters + string.digits + k = ''.join([allowed_chars[random.randrange(0, len(allowed_chars))] for i in range(4)]) tokens.append( - Key(base.TokValueLiteral(os.urandom(4))) + Key(base.TokValueLiteral(k)) ) return self.__class__( [i.resolve(settings, self) for i in tokens] diff --git a/pathod/utils.py b/pathod/utils.py index 1e5bd9a4..d1e2dd00 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -1,5 +1,6 @@ import os import sys +import netlib.utils SIZE_UNITS = dict( @@ -75,27 +76,7 @@ def escape_unprintables(s): return s -class Data(object): - - def __init__(self, name): - m = __import__(name) - dirname, _ = os.path.split(m.__file__) - self.dirname = os.path.abspath(dirname) - - def path(self, path): - """ - Returns a path to the package data housed at 'path' under this - module.Path can be a path to a file, or to a directory. - - This function will raise ValueError if the path does not exist. - """ - fullpath = os.path.join(self.dirname, path) - if not os.path.exists(fullpath): - raise ValueError("dataPath: %s does not exist." % fullpath) - return fullpath - - -data = Data(__name__) +data = netlib.utils.Data(__name__) def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover diff --git a/pathod/version.py b/pathod/version.py index 63f60a8d..2da7637d 100644 --- a/pathod/version.py +++ b/pathod/version.py @@ -1,3 +1,6 @@ from __future__ import (absolute_import, print_function, division) -from netlib.version import *
\ No newline at end of file +from netlib.version import VERSION, IVERSION + +NAME = "pathod" +NAMEVERSION = NAME + " " + VERSION @@ -67,10 +67,9 @@ setup( "construct>=2.5.2, <2.6", "cryptography>=1.3,<1.4", "Flask>=0.10.1, <0.11", - "h2>=2.1.2, <3.0", - "hpack>=2.1.0, <3.0", + "h2>=2.3.1, <3", "html2text>=2016.1.8, <=2016.4.2", - "hyperframe>=3.2.0, <4.1", + "hyperframe>=4.0.1, <5", "lxml>=3.5.0, <3.7", "Pillow>=3.2, <3.3", "passlib>=1.6.5, <1.7", diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index b560d9a1..0c117306 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -5,11 +5,12 @@ from contextlib import contextmanager from mitmproxy import utils, script from mitmproxy.proxy import config +import netlib.utils from netlib import tutils as netutils from netlib.http import Headers from . import tservers, tutils -example_dir = utils.Data(__name__).path("../../examples") +example_dir = netlib.utils.Data(__name__).path("../../examples") class DummyContext(object): @@ -106,8 +107,8 @@ def test_modify_querystring(): def test_modify_response_body(): with tutils.raises(script.ScriptException): - with example("modify_response_body.py") as ex: - pass + with example("modify_response_body.py"): + assert True flow = tutils.tflow(resp=netutils.tresp(content="I <3 mitmproxy")) with example("modify_response_body.py mitmproxy rocks") as ex: @@ -125,7 +126,7 @@ def test_redirect_requests(): def test_har_extractor(): with tutils.raises(script.ScriptException): - with example("har_extractor.py") as ex: + with example("har_extractor.py"): pass times = dict( diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 145e91cf..b9c6a2f6 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -7,7 +7,7 @@ import netlib.utils from netlib import odict from netlib.http import Headers from mitmproxy import filt, controller, tnetstring, flow -from mitmproxy.exceptions import FlowReadException +from mitmproxy.exceptions import FlowReadException, ScriptException from mitmproxy.models import Error from mitmproxy.models import Flow from mitmproxy.models import HTTPFlow @@ -76,6 +76,21 @@ class TestStickyCookieState: googlekey = s.jar.keys()[0] assert len(s.jar[googlekey].keys()) == 2 + # Test setting of weird cookie keys + s = flow.StickyCookieState(filt.parse(".*")) + f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True) + cs = [ + "foo/bar=hello", + "foo:bar=world", + "foo@bar=fizz", + "foo,bar=buzz", + ] + for c in cs: + f.response.headers["Set-Cookie"] = c + s.handle_response(f) + googlekey = s.jar.keys()[0] + assert len(s.jar[googlekey].keys()) == len(cs) + # Test overwriting of a cookie value c1 = "somecookie=helloworld; Path=/" c2 = "somecookie=newvalue; Path=/" @@ -84,7 +99,7 @@ class TestStickyCookieState: s.handle_response(f) googlekey = s.jar.keys()[0] assert len(s.jar[googlekey].keys()) == 1 - assert s.jar[googlekey]["somecookie"].value == "newvalue" + assert s.jar[googlekey]["somecookie"].items()[0][1] == "newvalue" def test_handle_request(self): s, f = self._response("SSID=mooo", "www.google.com") @@ -747,12 +762,16 @@ class TestFlowMaster: def test_load_script(self): s = flow.State() fm = flow.FlowMaster(None, s) - assert not fm.load_script(tutils.test_data.path("scripts/a.py")) - assert not fm.load_script(tutils.test_data.path("scripts/a.py")) - assert not fm.unload_scripts() - assert fm.load_script("nonexistent") - assert "ValueError" in fm.load_script( - tutils.test_data.path("scripts/starterr.py")) + + fm.load_script(tutils.test_data.path("scripts/a.py")) + fm.load_script(tutils.test_data.path("scripts/a.py")) + fm.unload_scripts() + with tutils.raises(ScriptException): + fm.load_script("nonexistent") + try: + fm.load_script(tutils.test_data.path("scripts/starterr.py")) + except ScriptException as e: + assert "ValueError" in str(e) assert len(fm.scripts) == 0 def test_getset_ignore(self): @@ -779,7 +798,7 @@ class TestFlowMaster: def test_script_reqerr(self): s = flow.State() fm = flow.FlowMaster(None, s) - assert not fm.load_script(tutils.test_data.path("scripts/reqerr.py")) + fm.load_script(tutils.test_data.path("scripts/reqerr.py")) f = tutils.tflow() fm.handle_clientconnect(f.client_conn) assert fm.handle_request(f) @@ -787,7 +806,7 @@ class TestFlowMaster: def test_script(self): s = flow.State() fm = flow.FlowMaster(None, s) - assert not fm.load_script(tutils.test_data.path("scripts/all.py")) + fm.load_script(tutils.test_data.path("scripts/all.py")) f = tutils.tflow(resp=True) fm.handle_clientconnect(f.client_conn) @@ -799,7 +818,7 @@ class TestFlowMaster: fm.handle_response(f) assert fm.scripts[0].ns["log"][-1] == "response" # load second script - assert not fm.load_script(tutils.test_data.path("scripts/all.py")) + fm.load_script(tutils.test_data.path("scripts/all.py")) assert len(fm.scripts) == 2 fm.handle_clientdisconnect(f.server_conn) assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" @@ -808,7 +827,7 @@ class TestFlowMaster: # unload first script fm.unload_scripts() assert len(fm.scripts) == 0 - assert not fm.load_script(tutils.test_data.path("scripts/all.py")) + fm.load_script(tutils.test_data.path("scripts/all.py")) f.error = tutils.terr() fm.handle_error(f) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 75a8090f..035f07b7 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -1,152 +1,74 @@ import json from textwrap import dedent +import re import netlib.tutils from netlib.http import Headers from mitmproxy import flow_export from . import tutils -req_get = netlib.tutils.treq( - method='GET', - content='', -) -req_post = netlib.tutils.treq( - method='POST', - headers=None, -) +def clean_blanks(s): + return re.sub(r"^(\s+)$", "", s, flags=re.MULTILINE) -req_patch = netlib.tutils.treq( - method='PATCH', - path=b"/path?query=param", -) +def python_equals(testdata, text): + """ + Compare two bits of Python code, disregarding non-significant differences + like whitespace on blank lines and trailing space. + """ + d = open(tutils.test_data.path(testdata)).read() + assert clean_blanks(text).rstrip() == clean_blanks(d).rstrip() + + +req_get = lambda: netlib.tutils.treq(method='GET', content='') + +req_post = lambda: netlib.tutils.treq(method='POST', headers=None) + +req_patch = lambda: netlib.tutils.treq(method='PATCH', path=b"/path?query=param") -class TestExportCurlCommand(): +class TestExportCurlCommand(): def test_get(self): - flow = tutils.tflow(req=req_get) + flow = tutils.tflow(req=req_get()) result = """curl -H 'header:qvalue' -H 'content-length:7' 'http://address/path'""" assert flow_export.curl_command(flow) == result def test_post(self): - flow = tutils.tflow(req=req_post) + flow = tutils.tflow(req=req_post()) result = """curl -X POST 'http://address/path' --data-binary 'content'""" assert flow_export.curl_command(flow) == result def test_patch(self): - flow = tutils.tflow(req=req_patch) + flow = tutils.tflow(req=req_patch()) result = """curl -H 'header:qvalue' -H 'content-length:7' -X PATCH 'http://address/path?query=param' --data-binary 'content'""" assert flow_export.curl_command(flow) == result class TestExportPythonCode(): - def test_get(self): - flow = tutils.tflow(req=req_get) - result = dedent(""" - import requests - - url = 'http://address/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - response = requests.request( - method='GET', - url=url, - headers=headers, - ) - - print(response.text) - """).strip() - assert flow_export.python_code(flow) == result + flow = tutils.tflow(req=req_get()) + python_equals("test_flow_export/python_get.py", flow_export.python_code(flow)) def test_post(self): - flow = tutils.tflow(req=req_post) - result = dedent(""" - import requests - - url = 'http://address/path' - - data = '''content''' - - response = requests.request( - method='POST', - url=url, - data=data, - ) - - print(response.text) - """).strip() - assert flow_export.python_code(flow) == result + flow = tutils.tflow(req=req_post()) + python_equals("test_flow_export/python_post.py", flow_export.python_code(flow)) def test_post_json(self): - req_post.content = '{"name": "example", "email": "example@example.com"}' - req_post.headers = Headers(content_type="application/json") - flow = tutils.tflow(req=req_post) - result = dedent(""" - import requests - - url = 'http://address/path' - - headers = { - 'content-type': 'application/json', - } - - json = { - "name": "example", - "email": "example@example.com" - } - - response = requests.request( - method='POST', - url=url, - headers=headers, - json=json, - ) - - print(response.text) - """).strip() - assert flow_export.python_code(flow) == result + p = req_post() + p.content = '{"name": "example", "email": "example@example.com"}' + p.headers = Headers(content_type="application/json") + flow = tutils.tflow(req=p) + python_equals("test_flow_export/python_post_json.py", flow_export.python_code(flow)) def test_patch(self): - flow = tutils.tflow(req=req_patch) - result = dedent(""" - import requests - - url = 'http://address/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - params = { - 'query': 'param', - } - - data = '''content''' - - response = requests.request( - method='PATCH', - url=url, - headers=headers, - params=params, - data=data, - ) - - print(response.text) - """).strip() - assert flow_export.python_code(flow) == result + flow = tutils.tflow(req=req_patch()) + python_equals("test_flow_export/python_patch.py", flow_export.python_code(flow)) class TestRawRequest(): - def test_get(self): - flow = tutils.tflow(req=req_get) + flow = tutils.tflow(req=req_get()) result = dedent(""" GET /path HTTP/1.1\r header: qvalue\r @@ -157,18 +79,17 @@ class TestRawRequest(): assert flow_export.raw_request(flow) == result def test_post(self): - flow = tutils.tflow(req=req_post) + flow = tutils.tflow(req=req_post()) result = dedent(""" POST /path HTTP/1.1\r - content-type: application/json\r host: address:22\r \r - {"name": "example", "email": "example@example.com"} + content """).strip() assert flow_export.raw_request(flow) == result def test_patch(self): - flow = tutils.tflow(req=req_patch) + flow = tutils.tflow(req=req_patch()) result = dedent(""" PATCH /path?query=param HTTP/1.1\r header: qvalue\r @@ -179,212 +100,50 @@ class TestRawRequest(): """).strip() assert flow_export.raw_request(flow) == result -class TestExportLocustCode(): +class TestExportLocustCode(): def test_get(self): - flow = tutils.tflow(req=req_get) - result = """ -from locust import HttpLocust, TaskSet, task - -class UserBehavior(TaskSet): - def on_start(self): - ''' on_start is called when a Locust start before any task is scheduled ''' - self.path() - - @task() - def path(self): - url = self.locust.host + '/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - self.response = self.client.request( - method='GET', - url=url, - headers=headers, - ) - - ### Additional tasks can go here ### - - -class WebsiteUser(HttpLocust): - task_set = UserBehavior - min_wait = 1000 - max_wait = 3000 - """.strip() - - assert flow_export.locust_code(flow) == result + flow = tutils.tflow(req=req_get()) + python_equals("test_flow_export/locust_get.py", flow_export.locust_code(flow)) def test_post(self): - req_post.content = '''content''' - req_post.headers = '' - flow = tutils.tflow(req=req_post) - result = """ -from locust import HttpLocust, TaskSet, task - -class UserBehavior(TaskSet): - def on_start(self): - ''' on_start is called when a Locust start before any task is scheduled ''' - self.path() - - @task() - def path(self): - url = self.locust.host + '/path' - - data = '''content''' - - self.response = self.client.request( - method='POST', - url=url, - data=data, - ) - - ### Additional tasks can go here ### - - -class WebsiteUser(HttpLocust): - task_set = UserBehavior - min_wait = 1000 - max_wait = 3000 - - """.strip() - - assert flow_export.locust_code(flow) == result - + p = req_post() + p.content = '''content''' + p.headers = '' + flow = tutils.tflow(req=p) + python_equals("test_flow_export/locust_post.py", flow_export.locust_code(flow)) def test_patch(self): - flow = tutils.tflow(req=req_patch) - result = """ -from locust import HttpLocust, TaskSet, task - -class UserBehavior(TaskSet): - def on_start(self): - ''' on_start is called when a Locust start before any task is scheduled ''' - self.path() - - @task() - def path(self): - url = self.locust.host + '/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - params = { - 'query': 'param', - } - - data = '''content''' - - self.response = self.client.request( - method='PATCH', - url=url, - headers=headers, - params=params, - data=data, - ) - - ### Additional tasks can go here ### - - -class WebsiteUser(HttpLocust): - task_set = UserBehavior - min_wait = 1000 - max_wait = 3000 - - """.strip() - - assert flow_export.locust_code(flow) == result + flow = tutils.tflow(req=req_patch()) + python_equals("test_flow_export/locust_patch.py", flow_export.locust_code(flow)) class TestExportLocustTask(): - def test_get(self): - flow = tutils.tflow(req=req_get) - result = ' ' + """ - @task() - def path(self): - url = self.locust.host + '/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - self.response = self.client.request( - method='GET', - url=url, - headers=headers, - ) - """.strip() + '\n' - - assert flow_export.locust_task(flow) == result + flow = tutils.tflow(req=req_get()) + python_equals("test_flow_export/locust_task_get.py", flow_export.locust_task(flow)) def test_post(self): - flow = tutils.tflow(req=req_post) - result = ' ' + """ - @task() - def path(self): - url = self.locust.host + '/path' - - data = '''content''' - - self.response = self.client.request( - method='POST', - url=url, - data=data, - ) - """.strip() + '\n' - - assert flow_export.locust_task(flow) == result - + flow = tutils.tflow(req=req_post()) + python_equals("test_flow_export/locust_task_post.py", flow_export.locust_task(flow)) def test_patch(self): - flow = tutils.tflow(req=req_patch) - result = ' ' + """ - @task() - def path(self): - url = self.locust.host + '/path' - - headers = { - 'header': 'qvalue', - 'content-length': '7', - } - - params = { - 'query': 'param', - } - - data = '''content''' - - self.response = self.client.request( - method='PATCH', - url=url, - headers=headers, - params=params, - data=data, - ) - """.strip() + '\n' - - assert flow_export.locust_task(flow) == result + flow = tutils.tflow(req=req_patch()) + python_equals("test_flow_export/locust_task_patch.py", flow_export.locust_task(flow)) class TestIsJson(): - def test_empty(self): - assert flow_export.is_json(None, None) == False + assert flow_export.is_json(None, None) is False def test_json_type(self): headers = Headers(content_type="application/json") - assert flow_export.is_json(headers, "foobar") == False + assert flow_export.is_json(headers, "foobar") is False def test_valid(self): headers = Headers(content_type="application/foobar") j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}') - assert j == False + assert j is False def test_valid(self): headers = Headers(content_type="application/json") diff --git a/test/mitmproxy/test_flow_export/locust_get.py b/test/mitmproxy/test_flow_export/locust_get.py new file mode 100644 index 00000000..72d5932a --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_get.py @@ -0,0 +1,29 @@ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.path() + + @task() + def path(self): + url = self.locust.host + '/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + self.response = self.client.request( + method='GET', + url=url, + headers=headers, + ) + + ### Additional tasks can go here ### + + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait = 1000 + max_wait = 3000 diff --git a/test/mitmproxy/test_flow_export/locust_patch.py b/test/mitmproxy/test_flow_export/locust_patch.py new file mode 100644 index 00000000..f64e0857 --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_patch.py @@ -0,0 +1,37 @@ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.path() + + @task() + def path(self): + url = self.locust.host + '/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + params = { + 'query': 'param', + } + + data = '''content''' + + self.response = self.client.request( + method='PATCH', + url=url, + headers=headers, + params=params, + data=data, + ) + + ### Additional tasks can go here ### + + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait = 1000 + max_wait = 3000 diff --git a/test/mitmproxy/test_flow_export/locust_post.py b/test/mitmproxy/test_flow_export/locust_post.py new file mode 100644 index 00000000..df23476a --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_post.py @@ -0,0 +1,26 @@ +from locust import HttpLocust, TaskSet, task + +class UserBehavior(TaskSet): + def on_start(self): + ''' on_start is called when a Locust start before any task is scheduled ''' + self.path() + + @task() + def path(self): + url = self.locust.host + '/path' + + data = '''content''' + + self.response = self.client.request( + method='POST', + url=url, + data=data, + ) + + ### Additional tasks can go here ### + + +class WebsiteUser(HttpLocust): + task_set = UserBehavior + min_wait = 1000 + max_wait = 3000 diff --git a/test/mitmproxy/test_flow_export/locust_task_get.py b/test/mitmproxy/test_flow_export/locust_task_get.py new file mode 100644 index 00000000..76f144fa --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_task_get.py @@ -0,0 +1,14 @@ + @task() + def path(self): + url = self.locust.host + '/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + self.response = self.client.request( + method='GET', + url=url, + headers=headers, + ) diff --git a/test/mitmproxy/test_flow_export/locust_task_patch.py b/test/mitmproxy/test_flow_export/locust_task_patch.py new file mode 100644 index 00000000..d425209c --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_task_patch.py @@ -0,0 +1,22 @@ + @task() + def path(self): + url = self.locust.host + '/path' + + headers = { + 'header': 'qvalue', + 'content-length': '7', + } + + params = { + 'query': 'param', + } + + data = '''content''' + + self.response = self.client.request( + method='PATCH', + url=url, + headers=headers, + params=params, + data=data, + ) diff --git a/test/mitmproxy/test_flow_export/locust_task_post.py b/test/mitmproxy/test_flow_export/locust_task_post.py new file mode 100644 index 00000000..989df455 --- /dev/null +++ b/test/mitmproxy/test_flow_export/locust_task_post.py @@ -0,0 +1,11 @@ + @task() + def path(self): + url = self.locust.host + '/path' + + data = '''content''' + + self.response = self.client.request( + method='POST', + url=url, + data=data, + ) diff --git a/test/mitmproxy/test_flow_export/python_get.py b/test/mitmproxy/test_flow_export/python_get.py new file mode 100644 index 00000000..ee3f48eb --- /dev/null +++ b/test/mitmproxy/test_flow_export/python_get.py @@ -0,0 +1,16 @@ +import requests + +url = 'http://address/path' + +headers = { + 'header': 'qvalue', + 'content-length': '7', +} + +response = requests.request( + method='GET', + url=url, + headers=headers, +) + +print(response.text) diff --git a/test/mitmproxy/test_flow_export/python_patch.py b/test/mitmproxy/test_flow_export/python_patch.py new file mode 100644 index 00000000..159e802f --- /dev/null +++ b/test/mitmproxy/test_flow_export/python_patch.py @@ -0,0 +1,24 @@ +import requests + +url = 'http://address/path' + +headers = { + 'header': 'qvalue', + 'content-length': '7', +} + +params = { + 'query': 'param', +} + +data = '''content''' + +response = requests.request( + method='PATCH', + url=url, + headers=headers, + params=params, + data=data, +) + +print(response.text) diff --git a/test/mitmproxy/test_flow_export/python_post.py b/test/mitmproxy/test_flow_export/python_post.py new file mode 100644 index 00000000..b13f6441 --- /dev/null +++ b/test/mitmproxy/test_flow_export/python_post.py @@ -0,0 +1,13 @@ +import requests + +url = 'http://address/path' + +data = '''content''' + +response = requests.request( + method='POST', + url=url, + data=data, +) + +print(response.text) diff --git a/test/mitmproxy/test_flow_export/python_post_json.py b/test/mitmproxy/test_flow_export/python_post_json.py new file mode 100644 index 00000000..7e105bf6 --- /dev/null +++ b/test/mitmproxy/test_flow_export/python_post_json.py @@ -0,0 +1,21 @@ +import requests + +url = 'http://address/path' + +headers = { + 'content-type': 'application/json', +} + +json = { + "name": "example", + "email": "example@example.com" +} + +response = requests.request( + method='POST', + url=url, + headers=headers, + json=json, +) + +print(response.text) diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 1da140d8..c3950975 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -1,3 +1,5 @@ +# coding=utf-8 + from __future__ import (absolute_import, print_function, division) import OpenSSL @@ -36,7 +38,7 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): class handler(netlib.tcp.BaseHandler): def handle(self): - h2_conn = h2.connection.H2Connection(client_side=False) + h2_conn = h2.connection.H2Connection(client_side=False, header_encoding=False) preamble = self.rfile.read(24) h2_conn.initiate_connection() @@ -122,7 +124,7 @@ class _Http2TestBase(object): client.convert_to_ssl(alpn_protos=[b'h2']) - h2_conn = h2.connection.H2Connection(client_side=True) + h2_conn = h2.connection.H2Connection(client_side=True, header_encoding=False) h2_conn.initiate_connection() client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() @@ -160,15 +162,19 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): if isinstance(event, h2.events.ConnectionTerminated): return False elif isinstance(event, h2.events.RequestReceived): - h2_conn.send_headers(1, [ + assert ('client-foo', 'client-bar-1') in event.headers + assert ('client-foo', 'client-bar-2') in event.headers + + h2_conn.send_headers(event.stream_id, [ (':status', '200'), - ('foo', 'bar'), + ('server-foo', 'server-bar'), + ('föo', 'bär'), + ('X-Stream-ID', str(event.stream_id)), ]) - h2_conn.send_data(1, b'foobar') - h2_conn.end_stream(1) + h2_conn.send_data(event.stream_id, b'foobar') + h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() - return True def test_simple(self): @@ -179,6 +185,8 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): (':method', 'GET'), (':scheme', 'https'), (':path', '/'), + ('ClIeNt-FoO', 'client-bar-1'), + ('ClIeNt-FoO', 'client-bar-2'), ], body='my request body echoed back to me') done = False @@ -200,7 +208,8 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): assert len(self.master.state.flows) == 1 assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.headers['foo'] == 'bar' + assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' + assert self.master.state.flows[0].response.headers['föo'] == 'bär' assert self.master.state.flows[0].response.body == b'foobar' @@ -425,3 +434,51 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): assert len(bodies) >= 1 assert b'regular_stream' in bodies # the other two bodies might not be transmitted before the reset + +@requires_alpn +class TestConnectionLost(_Http2TestBase, _Http2ServerBase): + + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class() + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(1, [(':status', '200')]) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return False + + def test_connection_lost(self): + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, stream_id=1, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('foo', 'bar') + ]) + + done = False + ended_streams = 0 + pushed_streams = 0 + responses = 0 + while not done: + try: + raw = b''.join(http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except: + break + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + if len(self.master.state.flows) == 1: + assert self.master.state.flows[0].response is None diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 8843ee62..454736d4 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -285,8 +285,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.master.set_stream_large_bodies(None) def test_stream_modify(self): - self.master.load_script( - tutils.test_data.path("scripts/stream_modify.py")) + self.master.load_script(tutils.test_data.path("scripts/stream_modify.py")) d = self.pathod('200:b"foo"') assert d.content == "bar" self.master.unload_scripts() @@ -511,8 +510,7 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): ssl = False def test_tcp_stream_modify(self): - self.master.load_script( - tutils.test_data.path("scripts/tcp_stream_modify.py")) + self.master.load_script(tutils.test_data.path("scripts/tcp_stream_modify.py")) self._tcpproxy_on() d = self.pathod('200:b"foo"') diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index d51ac185..2dfd710e 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -8,6 +8,7 @@ from contextlib import contextmanager from unittest.case import SkipTest +import netlib.utils import netlib.tutils from mitmproxy import utils, controller from mitmproxy.models import ( @@ -163,4 +164,4 @@ def capture_stderr(command, *args, **kwargs): sys.stderr = out -test_data = utils.Data(__name__) +test_data = netlib.utils.Data(__name__) diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py index 3b520a44..da28850f 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/netlib/http/test_cookies.py @@ -228,7 +228,16 @@ def test_refresh_cookie(): c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" assert "00:21:38" in cookies.refresh_set_cookie_header(c, 60) + c = "foo,bar" + with raises(ValueError): + cookies.refresh_set_cookie_header(c, 60) + # https://github.com/mitmproxy/mitmproxy/issues/773 c = ">=A" - with raises(ValueError): - cookies.refresh_set_cookie_header(c, 60)
\ No newline at end of file + assert cookies.refresh_set_cookie_header(c, 60) + + # https://github.com/mitmproxy/mitmproxy/issues/1118 + c = "foo:bar=bla" + assert cookies.refresh_set_cookie_header(c, 0) + c = "foo/bar=bla" + assert cookies.refresh_set_cookie_header(c, 0) diff --git a/test/netlib/http/test_request.py b/test/netlib/http/test_request.py index 50ad2d05..7ed6bd0f 100644 --- a/test/netlib/http/test_request.py +++ b/test/netlib/http/test_request.py @@ -107,6 +107,14 @@ class TestRequestUtils(object): with raises(ValueError): request.url = "not-a-url" + def test_url_options(self): + request = treq(method=b"OPTIONS", path=b"*") + assert request.url == "http://address:22" + + def test_url_authority(self): + request = treq(first_line_format="authority") + assert request.url == "address:22" + def test_pretty_host(self): request = treq() # Without host header @@ -140,6 +148,10 @@ class TestRequestUtils(object): request.headers["host"] = "other" assert request.pretty_url == "http://address:22/path" + def test_pretty_url_options(self): + request = treq(method=b"OPTIONS", path=b"*") + assert request.pretty_url == "http://address:22" + def test_pretty_url_authority(self): request = treq(first_line_format="authority") assert request.pretty_url == "address:22" @@ -160,7 +172,7 @@ class TestRequestUtils(object): def test_get_cookies_none(self): request = treq() request.headers = Headers() - assert len(request.cookies) == 0 + assert not request.cookies def test_get_cookies_single(self): request = treq() diff --git a/test/netlib/http/test_response.py b/test/netlib/http/test_response.py index a0c44d90..5440176c 100644 --- a/test/netlib/http/test_response.py +++ b/test/netlib/http/test_response.py @@ -98,7 +98,7 @@ class TestResponseUtils(object): resp = tresp() v = resp.cookies v.add("foo", ["bar", ODictCaseless()]) - resp.set_cookies(v) + resp.cookies = v v = resp.cookies assert len(v) == 1 diff --git a/test/netlib/test_odict.py b/test/netlib/test_odict.py index f0985ef6..b6fd6401 100644 --- a/test/netlib/test_odict.py +++ b/test/netlib/test_odict.py @@ -27,16 +27,6 @@ class TestODict(object): b.set_state(state) assert b == od - def test_in_any(self): - od = odict.ODict() - od["one"] = ["atwoa", "athreea"] - assert od.in_any("one", "two") - assert od.in_any("one", "three") - assert not od.in_any("one", "four") - assert not od.in_any("nonexistent", "foo") - assert not od.in_any("one", "TWO") - assert od.in_any("one", "TWO", True) - def test_iter(self): od = odict.ODict() assert not [i for i in od] diff --git a/test/netlib/test_utils.py b/test/netlib/test_utils.py index be2a59fc..1d8f7b0f 100644 --- a/test/netlib/test_utils.py +++ b/test/netlib/test_utils.py @@ -1,3 +1,4 @@ +# coding=utf-8 from netlib import utils, tutils from netlib.http import Headers @@ -170,3 +171,17 @@ class TestSerializable: def test_safe_subn(): assert utils.safe_subn("foo", u"bar", "\xc2foo") + + +def test_bytes_to_escaped_str(): + assert utils.bytes_to_escaped_str(b"foo") == "foo" + assert utils.bytes_to_escaped_str(b"\b") == r"\x08" + assert utils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" + assert utils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" + + +def test_escaped_str_to_bytes(): + assert utils.escaped_str_to_bytes("foo") == b"foo" + assert utils.escaped_str_to_bytes(r"\x08") == b"\b" + assert utils.escaped_str_to_bytes(r"&!?=\\)") == br"&!?=\)" + assert utils.escaped_str_to_bytes(r"ü") == b'\xc3\xbc' diff --git a/test/pathod/test_language_websocket.py b/test/pathod/test_language_websocket.py index f1105dfe..29155dba 100644 --- a/test/pathod/test_language_websocket.py +++ b/test/pathod/test_language_websocket.py @@ -97,7 +97,7 @@ class TestWebsocketFrame: assert self.fr("wf:r'foo'").payload == "foo" - def test_construction(self): + def test_construction_2(self): # Simple server frame frm = self.fr("wf:b'foo'") assert not frm.header.mask diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 8d0f92ac..4e8c89c5 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -211,7 +211,7 @@ class TestDaemon(_TestDaemon): c.stop() @skip_windows - @pytest.mark.xfail + @pytest.mark.skip(reason="race condition") def test_wait_finish(self): c = pathoc.Pathoc( ("127.0.0.1", self.d.port), diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 7583148b..05a3962e 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -129,7 +129,7 @@ class CommonTests(tutils.DaemonTests): l = self.d.last_log() # FIXME: Other binary data elements - @pytest.mark.xfail + @pytest.mark.skip(reason="race condition") def test_sizelimit(self): r = self.get("200:b@1g") assert r.status_code == 800 @@ -143,7 +143,7 @@ class CommonTests(tutils.DaemonTests): def test_info(self): assert tuple(self.d.info()["version"]) == version.IVERSION - @pytest.mark.xfail + @pytest.mark.skip(reason="race condition") def test_logs(self): assert self.d.clear_log() assert not self.d.last_log() @@ -223,7 +223,7 @@ class CommonTests(tutils.DaemonTests): ) assert r[1].payload == "test" - @pytest.mark.xfail + @pytest.mark.skip(reason="race condition") def test_websocket_frame_reflect_error(self): r, _ = self.pathoc( ["ws:/p/", "wf:-mask:knone:f'wf:b@10':i13,'a'"], @@ -233,6 +233,7 @@ class CommonTests(tutils.DaemonTests): # FIXME: Race Condition? assert "Parse error" in self.d.text_log() + @pytest.mark.skip(reason="race condition") def test_websocket_frame_disconnect_error(self): self.pathoc(["ws:/p/", "wf:b@10:d3"], ws_read_limit=0) assert self.d.last_log() diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index 9739afde..f6ed3efb 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -116,7 +116,7 @@ tmpdir = netlib.tutils.tmpdir raises = netlib.tutils.raises -test_data = utils.Data(__name__) +test_data = netlib.utils.Data(__name__) def render(r, settings=language.Settings()): diff --git a/web/src/css/header.less b/web/src/css/header.less index 8fa5e37f..065471d1 100644 --- a/web/src/css/header.less +++ b/web/src/css/header.less @@ -30,4 +30,8 @@ header { max-height: 500px; overflow-y: auto; } +} + +.menu .btn { + margin: 2px 2px 2px 2px; }
\ No newline at end of file diff --git a/web/src/js/components/common.js b/web/src/js/components/common.js index ad97ab38..21ca454f 100644 --- a/web/src/js/components/common.js +++ b/web/src/js/components/common.js @@ -131,4 +131,16 @@ export var Splitter = React.createClass({ </div> ); } -});
\ No newline at end of file +}); + +export const ToggleComponent = (props) => + <div + className={"btn " + (props.checked ? "btn-primary" : "btn-default")} + onClick={props.onToggleChanged}> + <span><i className={"fa " + (props.checked ? "fa-check-square-o" : "fa-square-o")}></i> {props.name}</span> + </div> + +ToggleComponent.propTypes = { + name: React.PropTypes.string.isRequired, + onToggleChanged: React.PropTypes.func.isRequired +}
\ No newline at end of file diff --git a/web/src/js/components/header.js b/web/src/js/components/header.js index b4110851..226cb61f 100644 --- a/web/src/js/components/header.js +++ b/web/src/js/components/header.js @@ -4,7 +4,7 @@ import $ from "jquery"; import Filt from "../filt/filt.js"; import {Key} from "../utils.js"; -import {Router} from "./common.js"; +import {Router, ToggleComponent} from "./common.js"; import {SettingsActions, FlowActions} from "../actions.js"; import {Query} from "../actions.js"; @@ -227,7 +227,6 @@ var ViewMenu = React.createClass({ mixins: [Router], toggleEventLog: function () { var d = {}; - if (this.getQuery()[Query.SHOW_EVENTLOG]) { d[Query.SHOW_EVENTLOG] = undefined; } else { @@ -235,24 +234,63 @@ var ViewMenu = React.createClass({ } this.updateLocation(undefined, d); + console.log('toggleevent'); }, render: function () { - var showEventLog = this.getQuery()[Query.SHOW_EVENTLOG]; - return ( - <div> - <button - className={"btn " + (showEventLog ? "btn-primary" : "btn-default")} - onClick={this.toggleEventLog}> - <i className="fa fa-database"></i> - Show Eventlog - </button> - <span> </span> - </div> - ); + var showEventLog = this.getQuery()[Query.SHOW_EVENTLOG]; + return ( + <div> + <ToggleComponent + checked={showEventLog} + name = "Show Eventlog" + onToggleChanged={this.toggleEventLog}/> + </div> + ); } }); +class OptionMenu extends React.Component{ + static title = "Options"; + constructor(props){ + super(props); + this.state = { + options : + [ + {name: "--host", checked: true}, + {name: "--no-upstream-cert", checked: false}, + {name: "--http2", checked: false}, + {name: "--anticache", checked: false}, + {name: "--anticomp", checked: false}, + {name: "--stickycookie", checked: true}, + {name: "--stickyauth", checked: false}, + {name: "--stream", checked: false} + ] + } + } + setOption(entry){ + console.log(entry.name);//TODO: get options from outside and remove state + entry.checked = !entry.checked; + this.setState({options: this.state.options}); + } + render() { + return ( + <div> + {this.state.options.map((entry, i) => { + return ( + <ToggleComponent + key={i} + checked={entry.checked} + name = {entry.name} + onToggleChanged={() => this.setOption(entry)}/> + ); + })} + </div> + ); + } +} + + var ReportsMenu = React.createClass({ statics: { title: "Visualization", @@ -349,7 +387,7 @@ var FileMenu = React.createClass({ }); -var header_entries = [MainMenu, ViewMenu /*, ReportsMenu */]; +var header_entries = [MainMenu, ViewMenu, OptionMenu /*, ReportsMenu */]; export var Header = React.createClass({ @@ -380,7 +418,7 @@ export var Header = React.createClass({ href="#" className={className} onClick={this.handleClick.bind(this, entry)}> - { entry.title} + {entry.title} </a> ); }.bind(this)); |