diff options
| -rw-r--r-- | .travis.yml | 4 | ||||
| -rw-r--r-- | README.rst | 45 | ||||
| -rwxr-xr-x | dev.sh | 5 | ||||
| -rw-r--r-- | mitmproxy/console/__init__.py | 33 | ||||
| -rw-r--r-- | mitmproxy/console/flowlist.py | 7 | ||||
| -rw-r--r-- | mitmproxy/console/statusbar.py | 4 | ||||
| -rw-r--r-- | mitmproxy/dump.py | 8 | ||||
| -rw-r--r-- | mitmproxy/web/app.py | 23 | ||||
| -rw-r--r-- | mitmproxy/web/static/app.js | 362 | ||||
| -rw-r--r-- | mitmproxy/web/static/vendor.js | 231 | ||||
| -rw-r--r-- | netlib/tcp.py | 43 | ||||
| -rw-r--r-- | pathod/language/generators.py | 57 | ||||
| -rw-r--r-- | pathod/pathoc.py | 3 | ||||
| -rw-r--r-- | pathod/pathod.py | 18 | ||||
| -rw-r--r-- | pathod/protocols/websockets.py | 2 | ||||
| -rw-r--r-- | pathod/test.py | 49 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | test/pathod/test_app.py | 6 | ||||
| -rw-r--r-- | test/pathod/test_language_generators.py | 13 | ||||
| -rw-r--r-- | test/pathod/test_pathod.py | 22 | ||||
| -rw-r--r-- | test/pathod/test_test.py | 4 | ||||
| -rw-r--r-- | test/pathod/tutils.py | 48 | ||||
| -rw-r--r-- | web/package.json | 1 | ||||
| -rw-r--r-- | web/src/js/actions.js | 12 | ||||
| -rw-r--r-- | web/src/js/components/header.js | 37 | ||||
| -rw-r--r-- | web/src/js/store/store.js | 2 | ||||
| -rw-r--r-- | web/src/js/utils.js | 16 |
27 files changed, 650 insertions, 408 deletions
diff --git a/.travis.yml b/.travis.yml index 9d4dea88..5ec8b3bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ matrix: git: depth: 9999999 - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py" + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py ./test/pathod/test_language_generators.py" - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py" NO_ALPN=1 + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py ./test/pathod/test_language_generators.py" NO_ALPN=1 - python: 2.7 env: DOCS=1 script: 'cd docs && make html' @@ -3,19 +3,24 @@ mitmproxy |travis| |coveralls| |latest_release| |python_versions| -This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**. +This repository contains the **mitmproxy** and **pathod** projects, as well as +their shared networking library, **netlib**. -``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface. +``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console +interface. ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP. -``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards. +``pathoc`` and ``pathod`` are perverse HTTP client and server applications +designed to let you craft almost any conceivable HTTP request, including ones +that creatively violate the standards. Documentation & Help -------------------- -Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites. +Documentation, tutorials and precompiled binaries can be found on the mitmproxy +and pathod websites. |mitmproxy_site| |pathod_site| @@ -28,12 +33,19 @@ You can join our developer chat on Slack. |slack| +Installation +------------ + +The installation instructions are `here <http://docs.mitmproxy.org/en/stable/install.html>`_. +If you want to contribute changes, keep on reading. + + Hacking ------- To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with -virtualenv_ installed (you can find installation instructions for virtualenv here_). -Then do the following: +virtualenv_ installed (you can find installation instructions for virtualenv +here_). Then do the following: .. code-block:: text @@ -42,10 +54,11 @@ Then do the following: ./dev.sh -The *dev* script will create a virtualenv environment in a directory called "venv", -and install all mandatory and optional dependencies into it. -The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable", -so any changes to the source in the repository will be reflected live in the virtualenv. +The *dev* script will create a virtualenv environment in a directory called +"venv", and install all mandatory and optional dependencies into it. The +primary mitmproxy components - mitmproxy, netlib and pathod - are installed as +"editable", so any changes to the source in the repository will be reflected +live in the virtualenv. To confirm that you're up and running, activate the virtualenv, and run the mitmproxy test suite: @@ -56,9 +69,9 @@ mitmproxy test suite: py.test Note that the main executables for the project - ``mitmdump``, ``mitmproxy``, -``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the -virtualenv, they will be on your $PATH, and you can run them like any other -command: +``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the +virtualenv. After activating the virtualenv, they will be on your $PATH, and +you can run them like any other command: .. code-block:: text @@ -85,9 +98,9 @@ suite. The project tries to maintain 100% test coverage. Documentation ------------- -The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development -environment as described above. -After installation, you can render the documentation like this: +The mitmproxy documentation is build using Sphinx_, which is installed +automatically if you set up a development environment as described above. After +installation, you can render the documentation like this: .. code-block:: text @@ -1,5 +1,6 @@ #!/bin/sh set -e +set -x PYVERSION=$1 VENV="venv$1" @@ -8,8 +9,8 @@ echo "Creating dev environment in $VENV using Python $PYVERSION" python$PYVERSION -m virtualenv "$VENV" --always-copy . "$VENV/bin/activate" -pip$PYVERSION install -q -U pip setuptools -pip$PYVERSION install -q -r requirements.txt +pip$PYVERSION install -U pip setuptools +pip$PYVERSION install -r requirements.txt echo "" echo "* Virtualenv created in $VENV and all dependencies installed." diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py index 00fb4b1b..63692ec0 100644 --- a/mitmproxy/console/__init__.py +++ b/mitmproxy/console/__init__.py @@ -44,6 +44,8 @@ class ConsoleState(flow.State): self.default_body_view = contentviews.get("Auto") self.flowsettings = weakref.WeakKeyDictionary() self.last_search = None + self.last_filter = None + self.mark_filter = False def __setattr__(self, name, value): self.__dict__[name] = value @@ -117,6 +119,37 @@ class ConsoleState(flow.State): self.set_focus(self.focus) return ret + def filter_marked(self, m): + def actual_func(x): + if x.id in m: + return True + return False + return actual_func + + def enable_marked_filter(self): + self.last_filter = self.limit_txt + marked_flows = [] + for f in self.flows: + if self.flow_marked(f): + marked_flows.append(f.id) + if len(marked_flows) > 0: + f = self.filter_marked(marked_flows) + self.view._close() + self.view = flow.FlowView(self.flows, f) + self.focus = 0 + self.set_focus(self.focus) + self.mark_filter = True + + def disable_marked_filter(self): + if self.last_filter is None: + self.view = flow.FlowView(self.flows, None) + else: + self.set_limit(self.last_filter) + self.focus = 0 + self.set_focus(self.focus) + self.last_filter = None + self.mark_filter = False + def clear(self): marked_flows = [] for f in self.flows: diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index eb1e76fb..8c20c4b6 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -22,6 +22,7 @@ def _mkhelp(): ("l", "set limit filter pattern"), ("L", "load saved flows"), ("m", "toggle flow mark"), + ("M", "toggle marked flow view"), ("n", "create a new request"), ("P", "copy flow to clipboard"), ("r", "replay request"), @@ -198,6 +199,12 @@ class ConnectionItem(urwid.WidgetWrap): else: self.state.set_flow_marked(self.flow, True) signals.flowlist_change.send(self) + elif key == "M": + if self.state.mark_filter: + self.state.disable_marked_filter() + else: + self.state.enable_marked_filter() + signals.flowlist_change.send(self) elif key == "r": r = self.master.replay_request(self.flow) if r: diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index b3e1517f..af8089b6 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -168,6 +168,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "l")) r.append(":%s]" % self.master.state.limit_txt) + if self.master.state.mark_filter: + r.append("[") + r.append(("heading_key", "Marked Flows")) + r.append("]") if self.master.stickycookie_txt: r.append("[") r.append(("heading_key", "t")) diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index b1005ee7..cc6896ed 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -333,17 +333,15 @@ class DumpMaster(flow.FlowMaster): @controller.handler def request(self, f): - flow.FlowMaster.request(self, f) - self.state.delete_flow(f) + f = flow.FlowMaster.request(self, f) if f: - f.reply() + self.state.delete_flow(f) return f @controller.handler def response(self, f): - flow.FlowMaster.response(self, f) + f = flow.FlowMaster.response(self, f) if f: - f.reply() self._process_flow(f) return f diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 43b2bad1..dce85df3 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -8,6 +8,8 @@ import re import six import tornado.websocket +from io import BytesIO +from mitmproxy.flow import FlowWriter, FlowReader from mitmproxy import filt from mitmproxy import version @@ -159,6 +161,26 @@ class Flows(RequestHandler): data=[_strip_content(f.get_state()) for f in self.state.flows] )) +class DumpFlows(RequestHandler): + def get(self): + self.set_header("Content-Disposition", "attachment; filename=flows") + self.set_header("Content-Type", "application/octet-stream") + + bio = BytesIO() + fw = FlowWriter(bio) + for f in self.state.flows: + fw.add(f) + + self.write(bio.getvalue()) + bio.close() + + def post(self): + self.state.clear() + + content = self.request.files.values()[0][0]["body"] + bio = BytesIO(content) + self.state.load_flows(FlowReader(bio).stream()) + bio.close() class ClearAll(RequestHandler): @@ -356,6 +378,7 @@ class Application(tornado.web.Application): (r"/updates", ClientConnection), (r"/events", Events), (r"/flows", Flows), + (r"/flows/dump", DumpFlows), (r"/flows/accept", AcceptFlows), (r"/flows/(?P<flow_id>[0-9a-f\-]+)", FlowHandler), (r"/flows/(?P<flow_id>[0-9a-f\-]+)/accept", AcceptFlow), diff --git a/mitmproxy/web/static/app.js b/mitmproxy/web/static/app.js index e1e4b22b..b78dd337 100644 --- a/mitmproxy/web/static/app.js +++ b/mitmproxy/web/static/app.js @@ -301,235 +301,6 @@ function isUndefined(arg) { },{}],2:[function(require,module,exports){ "use strict"; -function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } - -function _typeof(obj) { return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj; } - -var repeat = function repeat(str, times) { - return new Array(times + 1).join(str); -}; -var pad = function pad(num, maxLength) { - return repeat("0", maxLength - num.toString().length) + num; -}; -var formatTime = function formatTime(time) { - return "@ " + pad(time.getHours(), 2) + ":" + pad(time.getMinutes(), 2) + ":" + pad(time.getSeconds(), 2) + "." + pad(time.getMilliseconds(), 3); -}; - -// Use the new performance api to get better precision if available -var timer = typeof performance !== "undefined" && typeof performance.now === "function" ? performance : Date; - -/** - * parse the level option of createLogger - * - * @property {string | function | object} level - console[level] - * @property {object} action - * @property {array} payload - * @property {string} type - */ - -function getLogLevel(level, action, payload, type) { - switch (typeof level === "undefined" ? "undefined" : _typeof(level)) { - case "object": - return typeof level[type] === "function" ? level[type].apply(level, _toConsumableArray(payload)) : level[type]; - case "function": - return level(action); - default: - return level; - } -} - -/** - * Creates logger with followed options - * - * @namespace - * @property {object} options - options for logger - * @property {string | function | object} options.level - console[level] - * @property {boolean} options.duration - print duration of each action? - * @property {boolean} options.timestamp - print timestamp with each action? - * @property {object} options.colors - custom colors - * @property {object} options.logger - implementation of the `console` API - * @property {boolean} options.logErrors - should errors in action execution be caught, logged, and re-thrown? - * @property {boolean} options.collapsed - is group collapsed? - * @property {boolean} options.predicate - condition which resolves logger behavior - * @property {function} options.stateTransformer - transform state before print - * @property {function} options.actionTransformer - transform action before print - * @property {function} options.errorTransformer - transform error before print - */ - -function createLogger() { - var options = arguments.length <= 0 || arguments[0] === undefined ? {} : arguments[0]; - var _options$level = options.level; - var level = _options$level === undefined ? "log" : _options$level; - var _options$logger = options.logger; - var logger = _options$logger === undefined ? console : _options$logger; - var _options$logErrors = options.logErrors; - var logErrors = _options$logErrors === undefined ? true : _options$logErrors; - var collapsed = options.collapsed; - var predicate = options.predicate; - var _options$duration = options.duration; - var duration = _options$duration === undefined ? false : _options$duration; - var _options$timestamp = options.timestamp; - var timestamp = _options$timestamp === undefined ? true : _options$timestamp; - var transformer = options.transformer; - var _options$stateTransfo = options.stateTransformer; - var // deprecated - stateTransformer = _options$stateTransfo === undefined ? function (state) { - return state; - } : _options$stateTransfo; - var _options$actionTransf = options.actionTransformer; - var actionTransformer = _options$actionTransf === undefined ? function (actn) { - return actn; - } : _options$actionTransf; - var _options$errorTransfo = options.errorTransformer; - var errorTransformer = _options$errorTransfo === undefined ? function (error) { - return error; - } : _options$errorTransfo; - var _options$colors = options.colors; - var colors = _options$colors === undefined ? { - title: function title() { - return "#000000"; - }, - prevState: function prevState() { - return "#9E9E9E"; - }, - action: function action() { - return "#03A9F4"; - }, - nextState: function nextState() { - return "#4CAF50"; - }, - error: function error() { - return "#F20404"; - } - } : _options$colors; - - // exit if console undefined - - if (typeof logger === "undefined") { - return function () { - return function (next) { - return function (action) { - return next(action); - }; - }; - }; - } - - if (transformer) { - console.error("Option 'transformer' is deprecated, use stateTransformer instead"); - } - - var logBuffer = []; - function printBuffer() { - logBuffer.forEach(function (logEntry, key) { - var started = logEntry.started; - var startedTime = logEntry.startedTime; - var action = logEntry.action; - var prevState = logEntry.prevState; - var error = logEntry.error; - var took = logEntry.took; - var nextState = logEntry.nextState; - - var nextEntry = logBuffer[key + 1]; - if (nextEntry) { - nextState = nextEntry.prevState; - took = nextEntry.started - started; - } - // message - var formattedAction = actionTransformer(action); - var isCollapsed = typeof collapsed === "function" ? collapsed(function () { - return nextState; - }, action) : collapsed; - - var formattedTime = formatTime(startedTime); - var titleCSS = colors.title ? "color: " + colors.title(formattedAction) + ";" : null; - var title = "action " + (timestamp ? formattedTime : "") + " " + formattedAction.type + " " + (duration ? "(in " + took.toFixed(2) + " ms)" : ""); - - // render - try { - if (isCollapsed) { - if (colors.title) logger.groupCollapsed("%c " + title, titleCSS);else logger.groupCollapsed(title); - } else { - if (colors.title) logger.group("%c " + title, titleCSS);else logger.group(title); - } - } catch (e) { - logger.log(title); - } - - var prevStateLevel = getLogLevel(level, formattedAction, [prevState], "prevState"); - var actionLevel = getLogLevel(level, formattedAction, [formattedAction], "action"); - var errorLevel = getLogLevel(level, formattedAction, [error, prevState], "error"); - var nextStateLevel = getLogLevel(level, formattedAction, [nextState], "nextState"); - - if (prevStateLevel) { - if (colors.prevState) logger[prevStateLevel]("%c prev state", "color: " + colors.prevState(prevState) + "; font-weight: bold", prevState);else logger[prevStateLevel]("prev state", prevState); - } - - if (actionLevel) { - if (colors.action) logger[actionLevel]("%c action", "color: " + colors.action(formattedAction) + "; font-weight: bold", formattedAction);else logger[actionLevel]("action", formattedAction); - } - - if (error && errorLevel) { - if (colors.error) logger[errorLevel]("%c error", "color: " + colors.error(error, prevState) + "; font-weight: bold", error);else logger[errorLevel]("error", error); - } - - if (nextStateLevel) { - if (colors.nextState) logger[nextStateLevel]("%c next state", "color: " + colors.nextState(nextState) + "; font-weight: bold", nextState);else logger[nextStateLevel]("next state", nextState); - } - - try { - logger.groupEnd(); - } catch (e) { - logger.log("—— log end ——"); - } - }); - logBuffer.length = 0; - } - - return function (_ref) { - var getState = _ref.getState; - return function (next) { - return function (action) { - // exit early if predicate function returns false - if (typeof predicate === "function" && !predicate(getState, action)) { - return next(action); - } - - var logEntry = {}; - logBuffer.push(logEntry); - - logEntry.started = timer.now(); - logEntry.startedTime = new Date(); - logEntry.prevState = stateTransformer(getState()); - logEntry.action = action; - - var returnedValue = undefined; - if (logErrors) { - try { - returnedValue = next(action); - } catch (e) { - logEntry.error = errorTransformer(e); - } - } else { - returnedValue = next(action); - } - - logEntry.took = timer.now() - logEntry.started; - logEntry.nextState = stateTransformer(getState()); - - printBuffer(); - - if (logEntry.error) throw logEntry.error; - return returnedValue; - }; - }; - }; -} - -module.exports = createLogger; -},{}],3:[function(require,module,exports){ -"use strict"; - Object.defineProperty(exports, "__esModule", { value: true }); @@ -539,12 +310,10 @@ var _jquery = require("jquery"); var _jquery2 = _interopRequireDefault(_jquery); -var _lodash = require("lodash"); - -var _lodash2 = _interopRequireDefault(_lodash); - var _dispatcher = require("./dispatcher.js"); +var _utils = require("./utils.js"); + function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } var ActionTypes = exports.ActionTypes = { @@ -662,6 +431,18 @@ var FlowActions = exports.FlowActions = { }, clear: function clear() { _jquery2.default.post("/clear"); + }, + download: function download() { + return window.location = "/flows/dump"; + }, + + upload: function upload(file) { + var data = new FormData(); + data.append('file', file); + (0, _utils.fetchApi)("/flows/dump", { + method: 'post', + body: data + }); } }; @@ -671,7 +452,7 @@ var Query = exports.Query = { SHOW_EVENTLOG: "e" }; -},{"./dispatcher.js":23,"jquery":"jquery","lodash":"lodash"}],4:[function(require,module,exports){ +},{"./dispatcher.js":22,"./utils.js":31,"jquery":"jquery"}],3:[function(require,module,exports){ 'use strict'; var _react = require('react'); @@ -720,7 +501,7 @@ document.addEventListener('DOMContentLoaded', function () { ), document.getElementById("mitmproxy")); }); -},{"./components/proxyapp.js":21,"./connection":22,"./ducks/eventLog":24,"./ducks/index":25,"react":"react","react-dom":"react-dom","react-redux":"react-redux","redux":"redux","redux-logger":2}],5:[function(require,module,exports){ +},{"./components/proxyapp.js":20,"./connection":21,"./ducks/eventLog":23,"./ducks/index":24,"react":"react","react-dom":"react-dom","react-redux":"react-redux","redux":"redux","redux-logger":"redux-logger"}],4:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -939,7 +720,7 @@ ToggleInputButton.propTypes = { onToggleChanged: _react2.default.PropTypes.func.isRequired }; -},{"../utils.js":32,"lodash":"lodash","react":"react","react-dom":"react-dom"}],6:[function(require,module,exports){ +},{"../utils.js":31,"lodash":"lodash","react":"react","react-dom":"react-dom"}],5:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -1197,7 +978,7 @@ var ValueEditor = exports.ValueEditor = _react2.default.createClass({ } }); -},{"../utils.js":32,"react":"react","react-dom":"react-dom"}],7:[function(require,module,exports){ +},{"../utils.js":31,"react":"react","react-dom":"react-dom"}],6:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -1424,7 +1205,7 @@ var EventLogContainer = (0, _reactRedux.connect)(undefined, function (dispatch) exports.default = EventLogContainer; -},{"../ducks/eventLog":24,"./common":5,"./helpers/AutoScroll":17,"./helpers/VirtualScroll":18,"react":"react","react-dom":"react-dom","react-redux":"react-redux","shallowequal":"shallowequal"}],8:[function(require,module,exports){ +},{"../ducks/eventLog":23,"./common":4,"./helpers/AutoScroll":16,"./helpers/VirtualScroll":17,"react":"react","react-dom":"react-dom","react-redux":"react-redux","shallowequal":"shallowequal"}],7:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -1693,7 +1474,7 @@ var all_columns = [TLSColumn, IconColumn, PathColumn, MethodColumn, StatusColumn exports.default = all_columns; -},{"../flow/utils.js":29,"../utils.js":32,"react":"react"}],9:[function(require,module,exports){ +},{"../flow/utils.js":28,"../utils.js":31,"react":"react"}],8:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -1985,7 +1766,7 @@ FlowTable.defaultProps = { }; exports.default = (0, _AutoScroll2.default)(FlowTable); -},{"../utils.js":32,"./flowtable-columns.js":8,"./helpers/AutoScroll":17,"./helpers/VirtualScroll":18,"classnames":"classnames","lodash":"lodash","react":"react","react-dom":"react-dom","shallowequal":"shallowequal"}],10:[function(require,module,exports){ +},{"../utils.js":31,"./flowtable-columns.js":7,"./helpers/AutoScroll":16,"./helpers/VirtualScroll":17,"classnames":"classnames","lodash":"lodash","react":"react","react-dom":"react-dom","shallowequal":"shallowequal"}],9:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -2339,7 +2120,7 @@ var ContentView = _react2.default.createClass({ exports.default = ContentView; -},{"../../flow/utils.js":29,"../../utils.js":32,"lodash":"lodash","react":"react"}],11:[function(require,module,exports){ +},{"../../flow/utils.js":28,"../../utils.js":31,"lodash":"lodash","react":"react"}],10:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -2607,7 +2388,7 @@ var Details = _react2.default.createClass({ exports.default = Details; -},{"../../utils.js":32,"lodash":"lodash","react":"react"}],12:[function(require,module,exports){ +},{"../../utils.js":31,"lodash":"lodash","react":"react"}],11:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -2736,7 +2517,7 @@ var FlowView = _react2.default.createClass({ exports.default = FlowView; -},{"../prompt.js":20,"./details.js":11,"./messages.js":13,"./nav.js":14,"react":"react"}],13:[function(require,module,exports){ +},{"../prompt.js":19,"./details.js":10,"./messages.js":12,"./nav.js":13,"react":"react"}],12:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3101,7 +2882,7 @@ var Error = exports.Error = _react2.default.createClass({ } }); -},{"../../actions.js":3,"../../flow/utils.js":29,"../../utils.js":32,"../editor.js":6,"./contentview.js":10,"lodash":"lodash","react":"react","react-dom":"react-dom"}],14:[function(require,module,exports){ +},{"../../actions.js":2,"../../flow/utils.js":28,"../../utils.js":31,"../editor.js":5,"./contentview.js":9,"lodash":"lodash","react":"react","react-dom":"react-dom"}],13:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3182,7 +2963,7 @@ var Nav = _react2.default.createClass({ exports.default = Nav; -},{"../../actions.js":3,"react":"react"}],15:[function(require,module,exports){ +},{"../../actions.js":2,"react":"react"}],14:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3284,7 +3065,7 @@ function Footer(_ref) { ); } -},{"../utils.js":32,"./common.js":5,"react":"react"}],16:[function(require,module,exports){ +},{"../utils.js":31,"./common.js":4,"react":"react"}],15:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3718,18 +3499,27 @@ var FileMenu = _react2.default.createClass({ } }, handleOpenClick: function handleOpenClick(e) { + this.fileInput.click(); + e.preventDefault(); + }, + handleOpenFile: function handleOpenFile(e) { + if (e.target.files.length > 0) { + _actions.FlowActions.upload(e.target.files[0]); + this.fileInput.value = ""; + } e.preventDefault(); - console.error("unimplemented: handleOpenClick"); }, handleSaveClick: function handleSaveClick(e) { e.preventDefault(); - console.error("unimplemented: handleSaveClick"); + _actions.FlowActions.download(); }, handleShutdownClick: function handleShutdownClick(e) { e.preventDefault(); console.error("unimplemented: handleShutdownClick"); }, render: function render() { + var _this = this; + var fileMenuClass = "dropdown pull-left" + (this.state.showFileMenu ? " open" : ""); return _react2.default.createElement( @@ -3753,6 +3543,29 @@ var FileMenu = _react2.default.createClass({ "New" ) ), + _react2.default.createElement( + "li", + null, + _react2.default.createElement( + "a", + { href: "#", onClick: this.handleOpenClick }, + _react2.default.createElement("i", { className: "fa fa-fw fa-folder-open" }), + "Open..." + ), + _react2.default.createElement("input", { ref: function ref(_ref) { + return _this.fileInput = _ref; + }, className: "hidden", type: "file", onChange: this.handleOpenFile }) + ), + _react2.default.createElement( + "li", + null, + _react2.default.createElement( + "a", + { href: "#", onClick: this.handleSaveClick }, + _react2.default.createElement("i", { className: "fa fa-fw fa-floppy-o" }), + "Save..." + ) + ), _react2.default.createElement("li", { role: "presentation", className: "divider" }), _react2.default.createElement( "li", @@ -3827,7 +3640,7 @@ var Header = exports.Header = _react2.default.createClass({ } }); -},{"../actions.js":3,"../filt/filt.js":28,"../utils.js":32,"./common.js":5,"./eventlog":7,"jquery":"jquery","react":"react","react-dom":"react-dom","react-redux":"react-redux"}],17:[function(require,module,exports){ +},{"../actions.js":2,"../filt/filt.js":27,"../utils.js":31,"./common.js":4,"./eventlog":6,"jquery":"jquery","react":"react","react-dom":"react-dom","react-redux":"react-redux"}],16:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3893,7 +3706,7 @@ exports.default = function (Component) { }(Component), _class.displayName = Component.name, _temp), Component); }; -},{"react":"react","react-dom":"react-dom"}],18:[function(require,module,exports){ +},{"react":"react","react-dom":"react-dom"}],17:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -3972,7 +3785,7 @@ function calcVScroll(opts) { return { start: start, end: end, paddingTop: paddingTop, paddingBottom: paddingBottom }; } -},{}],19:[function(require,module,exports){ +},{}],18:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -4243,7 +4056,7 @@ var MainView = _react2.default.createClass({ exports.default = MainView; -},{"../actions.js":3,"../filt/filt.js":28,"../store/view.js":31,"../utils.js":32,"./common.js":5,"./flowtable.js":9,"./flowview/index.js":12,"react":"react"}],20:[function(require,module,exports){ +},{"../actions.js":2,"../filt/filt.js":27,"../store/view.js":30,"../utils.js":31,"./common.js":4,"./flowtable.js":8,"./flowview/index.js":11,"react":"react"}],19:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -4379,7 +4192,7 @@ var Prompt = _react2.default.createClass({ exports.default = Prompt; -},{"../utils.js":32,"lodash":"lodash","react":"react","react-dom":"react-dom"}],21:[function(require,module,exports){ +},{"../utils.js":31,"lodash":"lodash","react":"react","react-dom":"react-dom"}],20:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -4575,7 +4388,7 @@ var App = exports.App = _react2.default.createElement( ) ); -},{"../store/store.js":30,"../utils.js":32,"./common.js":5,"./eventlog.js":7,"./footer.js":15,"./header.js":16,"./mainview.js":19,"lodash":"lodash","react":"react","react-dom":"react-dom","react-redux":"react-redux","react-router":"react-router"}],22:[function(require,module,exports){ +},{"../store/store.js":29,"../utils.js":31,"./common.js":4,"./eventlog.js":6,"./footer.js":14,"./header.js":15,"./mainview.js":18,"lodash":"lodash","react":"react","react-dom":"react-dom","react-redux":"react-redux","react-router":"react-router"}],21:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -4621,7 +4434,7 @@ function Connection(url, dispatch) { return ws; } -},{"./actions.js":3,"./dispatcher.js":23,"./ducks/websocket":27}],23:[function(require,module,exports){ +},{"./actions.js":2,"./dispatcher.js":22,"./ducks/websocket":26}],22:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -4650,7 +4463,7 @@ AppDispatcher.dispatchServerAction = function (action) { this.dispatch(action); }; -},{"flux":"flux"}],24:[function(require,module,exports){ +},{"flux":"flux"}],23:[function(require,module,exports){ 'use strict'; Object.defineProperty(exports, "__esModule", { @@ -4734,7 +4547,7 @@ function addLogEntry(message) { }; } -},{"./list":26}],25:[function(require,module,exports){ +},{"./list":25}],24:[function(require,module,exports){ 'use strict'; Object.defineProperty(exports, "__esModule", { @@ -4760,7 +4573,7 @@ var rootReducer = (0, _redux.combineReducers)({ exports.default = rootReducer; -},{"./eventLog.js":24,"./websocket.js":27,"redux":"redux"}],26:[function(require,module,exports){ +},{"./eventLog.js":23,"./websocket.js":26,"redux":"redux"}],25:[function(require,module,exports){ 'use strict'; Object.defineProperty(exports, "__esModule", { @@ -4800,7 +4613,7 @@ function getList() { } } -},{}],27:[function(require,module,exports){ +},{}],26:[function(require,module,exports){ 'use strict'; Object.defineProperty(exports, "__esModule", { @@ -4841,7 +4654,7 @@ function disconnected() { return { type: DISCONNECTED }; } -},{}],28:[function(require,module,exports){ +},{}],27:[function(require,module,exports){ "use strict"; module.exports = function () { @@ -6745,7 +6558,7 @@ module.exports = function () { }; }(); -},{"../flow/utils.js":29}],29:[function(require,module,exports){ +},{"../flow/utils.js":28}],28:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -6879,7 +6692,7 @@ var parseHttpVersion = exports.parseHttpVersion = function parseHttpVersion(http }); }; -},{"jquery":"jquery","lodash":"lodash"}],30:[function(require,module,exports){ +},{"jquery":"jquery","lodash":"lodash"}],29:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -7013,7 +6826,7 @@ _lodash2.default.extend(LiveStoreMixin.prototype, { this._fetchxhr = _jquery2.default.getJSON("/" + this.type).done(function (message) { this.handle_fetch(message.data); }.bind(this)).fail(function () { - EventLogActions.add_event("Could not fetch " + this.type); + _actions.EventLogActions.add_event("Could not fetch " + this.type); }.bind(this)); } }, @@ -7066,7 +6879,7 @@ _lodash2.default.extend(EventLogStore.prototype, LiveListStore.prototype, { } }); -},{"../actions.js":3,"../dispatcher.js":23,"events":1,"jquery":"jquery","lodash":"lodash"}],31:[function(require,module,exports){ +},{"../actions.js":2,"../dispatcher.js":22,"events":1,"jquery":"jquery","lodash":"lodash"}],30:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { @@ -7196,14 +7009,18 @@ _lodash2.default.extend(StoreView.prototype, _events.EventEmitter.prototype, { } }); -},{"../utils.js":32,"events":1,"lodash":"lodash"}],32:[function(require,module,exports){ +},{"../utils.js":31,"events":1,"lodash":"lodash"}],31:[function(require,module,exports){ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.formatTimeStamp = exports.formatTimeDelta = exports.formatSize = exports.Key = undefined; + +var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; + exports.reverseString = reverseString; +exports.fetchApi = fetchApi; var _jquery = require("jquery"); @@ -7288,7 +7105,7 @@ function getCookie(name) { var r = document.cookie.match(new RegExp("\\b" + name + "=([^;]*)\\b")); return r ? r[1] : undefined; } -var xsrf = _jquery2.default.param({ _xsrf: getCookie("_xsrf") }); +var xsrf = "_xsrf=" + getCookie("_xsrf"); //Tornado XSRF Protection. _jquery2.default.ajaxPrefilter(function (options) { @@ -7311,7 +7128,18 @@ _jquery2.default.ajaxPrefilter(function (options) { alert(message); }); -},{"./actions.js":3,"jquery":"jquery","lodash":"lodash","react":"react"}]},{},[4]) +function fetchApi(url, options) { + if (url.indexOf("?") === -1) { + url += "?" + xsrf; + } else { + url += "&" + xsrf; + } + return fetch(url, _extends({}, options, { + credentials: 'same-origin' + })); +} + +},{"./actions.js":2,"jquery":"jquery","lodash":"lodash","react":"react"}]},{},[3]) //# sourceMappingURL=app.js.map diff --git a/mitmproxy/web/static/vendor.js b/mitmproxy/web/static/vendor.js index 03ff5871..61596f06 100644 --- a/mitmproxy/web/static/vendor.js +++ b/mitmproxy/web/static/vendor.js @@ -53352,7 +53352,236 @@ exports.createMemoryHistory = _createMemoryHistory3.default; module.exports = require('./lib/React'); -},{"./lib/React":102}],"redux":[function(require,module,exports){ +},{"./lib/React":102}],"redux-logger":[function(require,module,exports){ +"use strict"; + +function _toConsumableArray(arr) { if (Array.isArray(arr)) { for (var i = 0, arr2 = Array(arr.length); i < arr.length; i++) { arr2[i] = arr[i]; } return arr2; } else { return Array.from(arr); } } + +function _typeof(obj) { return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj; } + +var repeat = function repeat(str, times) { + return new Array(times + 1).join(str); +}; +var pad = function pad(num, maxLength) { + return repeat("0", maxLength - num.toString().length) + num; +}; +var formatTime = function formatTime(time) { + return "@ " + pad(time.getHours(), 2) + ":" + pad(time.getMinutes(), 2) + ":" + pad(time.getSeconds(), 2) + "." + pad(time.getMilliseconds(), 3); +}; + +// Use the new performance api to get better precision if available +var timer = typeof performance !== "undefined" && typeof performance.now === "function" ? performance : Date; + +/** + * parse the level option of createLogger + * + * @property {string | function | object} level - console[level] + * @property {object} action + * @property {array} payload + * @property {string} type + */ + +function getLogLevel(level, action, payload, type) { + switch (typeof level === "undefined" ? "undefined" : _typeof(level)) { + case "object": + return typeof level[type] === "function" ? level[type].apply(level, _toConsumableArray(payload)) : level[type]; + case "function": + return level(action); + default: + return level; + } +} + +/** + * Creates logger with followed options + * + * @namespace + * @property {object} options - options for logger + * @property {string | function | object} options.level - console[level] + * @property {boolean} options.duration - print duration of each action? + * @property {boolean} options.timestamp - print timestamp with each action? + * @property {object} options.colors - custom colors + * @property {object} options.logger - implementation of the `console` API + * @property {boolean} options.logErrors - should errors in action execution be caught, logged, and re-thrown? + * @property {boolean} options.collapsed - is group collapsed? + * @property {boolean} options.predicate - condition which resolves logger behavior + * @property {function} options.stateTransformer - transform state before print + * @property {function} options.actionTransformer - transform action before print + * @property {function} options.errorTransformer - transform error before print + */ + +function createLogger() { + var options = arguments.length <= 0 || arguments[0] === undefined ? {} : arguments[0]; + var _options$level = options.level; + var level = _options$level === undefined ? "log" : _options$level; + var _options$logger = options.logger; + var logger = _options$logger === undefined ? console : _options$logger; + var _options$logErrors = options.logErrors; + var logErrors = _options$logErrors === undefined ? true : _options$logErrors; + var collapsed = options.collapsed; + var predicate = options.predicate; + var _options$duration = options.duration; + var duration = _options$duration === undefined ? false : _options$duration; + var _options$timestamp = options.timestamp; + var timestamp = _options$timestamp === undefined ? true : _options$timestamp; + var transformer = options.transformer; + var _options$stateTransfo = options.stateTransformer; + var // deprecated + stateTransformer = _options$stateTransfo === undefined ? function (state) { + return state; + } : _options$stateTransfo; + var _options$actionTransf = options.actionTransformer; + var actionTransformer = _options$actionTransf === undefined ? function (actn) { + return actn; + } : _options$actionTransf; + var _options$errorTransfo = options.errorTransformer; + var errorTransformer = _options$errorTransfo === undefined ? function (error) { + return error; + } : _options$errorTransfo; + var _options$colors = options.colors; + var colors = _options$colors === undefined ? { + title: function title() { + return "#000000"; + }, + prevState: function prevState() { + return "#9E9E9E"; + }, + action: function action() { + return "#03A9F4"; + }, + nextState: function nextState() { + return "#4CAF50"; + }, + error: function error() { + return "#F20404"; + } + } : _options$colors; + + // exit if console undefined + + if (typeof logger === "undefined") { + return function () { + return function (next) { + return function (action) { + return next(action); + }; + }; + }; + } + + if (transformer) { + console.error("Option 'transformer' is deprecated, use stateTransformer instead"); + } + + var logBuffer = []; + function printBuffer() { + logBuffer.forEach(function (logEntry, key) { + var started = logEntry.started; + var startedTime = logEntry.startedTime; + var action = logEntry.action; + var prevState = logEntry.prevState; + var error = logEntry.error; + var took = logEntry.took; + var nextState = logEntry.nextState; + + var nextEntry = logBuffer[key + 1]; + if (nextEntry) { + nextState = nextEntry.prevState; + took = nextEntry.started - started; + } + // message + var formattedAction = actionTransformer(action); + var isCollapsed = typeof collapsed === "function" ? collapsed(function () { + return nextState; + }, action) : collapsed; + + var formattedTime = formatTime(startedTime); + var titleCSS = colors.title ? "color: " + colors.title(formattedAction) + ";" : null; + var title = "action " + (timestamp ? formattedTime : "") + " " + formattedAction.type + " " + (duration ? "(in " + took.toFixed(2) + " ms)" : ""); + + // render + try { + if (isCollapsed) { + if (colors.title) logger.groupCollapsed("%c " + title, titleCSS);else logger.groupCollapsed(title); + } else { + if (colors.title) logger.group("%c " + title, titleCSS);else logger.group(title); + } + } catch (e) { + logger.log(title); + } + + var prevStateLevel = getLogLevel(level, formattedAction, [prevState], "prevState"); + var actionLevel = getLogLevel(level, formattedAction, [formattedAction], "action"); + var errorLevel = getLogLevel(level, formattedAction, [error, prevState], "error"); + var nextStateLevel = getLogLevel(level, formattedAction, [nextState], "nextState"); + + if (prevStateLevel) { + if (colors.prevState) logger[prevStateLevel]("%c prev state", "color: " + colors.prevState(prevState) + "; font-weight: bold", prevState);else logger[prevStateLevel]("prev state", prevState); + } + + if (actionLevel) { + if (colors.action) logger[actionLevel]("%c action", "color: " + colors.action(formattedAction) + "; font-weight: bold", formattedAction);else logger[actionLevel]("action", formattedAction); + } + + if (error && errorLevel) { + if (colors.error) logger[errorLevel]("%c error", "color: " + colors.error(error, prevState) + "; font-weight: bold", error);else logger[errorLevel]("error", error); + } + + if (nextStateLevel) { + if (colors.nextState) logger[nextStateLevel]("%c next state", "color: " + colors.nextState(nextState) + "; font-weight: bold", nextState);else logger[nextStateLevel]("next state", nextState); + } + + try { + logger.groupEnd(); + } catch (e) { + logger.log("—— log end ——"); + } + }); + logBuffer.length = 0; + } + + return function (_ref) { + var getState = _ref.getState; + return function (next) { + return function (action) { + // exit early if predicate function returns false + if (typeof predicate === "function" && !predicate(getState, action)) { + return next(action); + } + + var logEntry = {}; + logBuffer.push(logEntry); + + logEntry.started = timer.now(); + logEntry.startedTime = new Date(); + logEntry.prevState = stateTransformer(getState()); + logEntry.action = action; + + var returnedValue = undefined; + if (logErrors) { + try { + returnedValue = next(action); + } catch (e) { + logEntry.error = errorTransformer(e); + } + } else { + returnedValue = next(action); + } + + logEntry.took = timer.now() - logEntry.started; + logEntry.nextState = stateTransformer(getState()); + + printBuffer(); + + if (logEntry.error) throw logEntry.error; + return returnedValue; + }; + }; + }; +} + +module.exports = createLogger; +},{}],"redux":[function(require,module,exports){ (function (process){ 'use strict'; diff --git a/netlib/tcp.py b/netlib/tcp.py index 914aa701..de12102e 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -6,6 +6,7 @@ import sys import threading import time import traceback +import contextlib import binascii from six.moves import range @@ -577,6 +578,12 @@ class _Connection(object): return context +@contextlib.contextmanager +def _closer(client): + yield + client.close() + + class TCPClient(_Connection): def __init__(self, address, source_address=None): @@ -708,6 +715,7 @@ class TCPClient(_Connection): self.connection = connection self.ip_address = Address(connection.getpeername()) self._makefile() + return _closer(self) def settimeout(self, n): self.connection.settimeout(n) @@ -833,6 +841,25 @@ class BaseHandler(_Connection): return b"" +class Counter: + def __init__(self): + self._count = 0 + self._lock = threading.Lock() + + @property + def count(self): + with self._lock: + return self._count + + def __enter__(self): + with self._lock: + self._count += 1 + + def __exit__(self, *args): + with self._lock: + self._count -= 1 + + class TCPServer(object): request_queue_size = 20 @@ -845,15 +872,17 @@ class TCPServer(object): self.socket.bind(self.address()) self.address = Address.wrap(self.socket.getsockname()) self.socket.listen(self.request_queue_size) + self.handler_counter = Counter() def connection_thread(self, connection, client_address): - client_address = Address(client_address) - try: - self.handle_client_connection(connection, client_address) - except: - self.handle_error(connection, client_address) - finally: - close_socket(connection) + with self.handler_counter: + client_address = Address(client_address) + try: + self.handle_client_connection(connection, client_address) + except: + self.handle_error(connection, client_address) + finally: + close_socket(connection) def serve_forever(self, poll_interval=0.1): self.__is_shut_down.clear() diff --git a/pathod/language/generators.py b/pathod/language/generators.py index a17e7052..9fff3082 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -2,17 +2,19 @@ import string import random import mmap +import six + DATATYPES = dict( - ascii_letters=string.ascii_letters, - ascii_lowercase=string.ascii_lowercase, - ascii_uppercase=string.ascii_uppercase, - digits=string.digits, - hexdigits=string.hexdigits, - octdigits=string.octdigits, - punctuation=string.punctuation, - whitespace=string.whitespace, - ascii=string.printable, - bytes="".join(chr(i) for i in range(256)) + ascii_letters=string.ascii_letters.encode(), + ascii_lowercase=string.ascii_lowercase.encode(), + ascii_uppercase=string.ascii_uppercase.encode(), + digits=string.digits.encode(), + hexdigits=string.hexdigits.encode(), + octdigits=string.octdigits.encode(), + punctuation=string.punctuation.encode(), + whitespace=string.whitespace.encode(), + ascii=string.printable.encode(), + bytes=bytes(bytearray(range(256))) ) @@ -35,16 +37,25 @@ class TransformGenerator(object): def __getitem__(self, x): d = self.gen.__getitem__(x) + if isinstance(x, slice): + return self.transform(x.start, d) return self.transform(x, d) - def __getslice__(self, a, b): - d = self.gen.__getslice__(a, b) - return self.transform(a, d) - def __repr__(self): return "'transform(%s)'" % self.gen +def rand_byte(chars): + """ + Return a random character as byte from a charset. + """ + # bytearray has consistent behaviour on both Python 2 and 3 + # while bytes does not + if six.PY2: + return random.choice(chars) + return bytes([random.choice(chars)]) + + class RandomGenerator(object): def __init__(self, dtype, length): @@ -55,12 +66,10 @@ class RandomGenerator(object): return self.length def __getitem__(self, x): - return random.choice(DATATYPES[self.dtype]) - - def __getslice__(self, a, b): - b = min(b, self.length) chars = DATATYPES[self.dtype] - return "".join(random.choice(chars) for x in range(a, b)) + if isinstance(x, slice): + return b"".join(rand_byte(chars) for _ in range(*x.indices(self.length))) + return rand_byte(chars) def __repr__(self): return "%s random from %s" % (self.length, self.dtype) @@ -70,17 +79,17 @@ class FileGenerator(object): def __init__(self, path): self.path = path - self.fp = file(path, "rb") + self.fp = open(path, "rb") self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ) def __len__(self): return len(self.map) def __getitem__(self, x): - return self.map.__getitem__(x) - - def __getslice__(self, a, b): - return self.map.__getslice__(a, b) + if isinstance(x, slice): + return self.map.__getitem__(x) + # A slice of length 1 returns a byte object (not an integer) + return self.map.__getitem__(slice(x, x + 1 or self.map.size())) def __repr__(self): return "<%s" % self.path diff --git a/pathod/pathoc.py b/pathod/pathoc.py index 2b7d053c..5cfb4591 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -286,7 +286,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2 and not self.ssl: raise NotImplementedError("HTTP2 without SSL is not supported.") - tcp.TCPClient.connect(self) + ret = tcp.TCPClient.connect(self) if connect_to: self.http_connect(connect_to) @@ -324,6 +324,7 @@ class Pathoc(tcp.TCPClient): if self.timeout: self.settimeout(self.timeout) + return ret def stop(self): if self.ws_framereader: diff --git a/pathod/pathod.py b/pathod/pathod.py index 7795df0e..0449c0c1 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -353,6 +353,8 @@ class Pathod(tcp.TCPServer): staticdir=self.staticdir ) + self.loglock = threading.Lock() + def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. @@ -403,8 +405,7 @@ class Pathod(tcp.TCPServer): def add_log(self, d): if not self.noapi: - lock = threading.Lock() - with lock: + with self.loglock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: @@ -413,17 +414,18 @@ class Pathod(tcp.TCPServer): return d["id"] def clear_log(self): - lock = threading.Lock() - with lock: + with self.loglock: self.log = [] def log_by_id(self, identifier): - for i in self.log: - if i["id"] == identifier: - return i + with self.loglock: + for i in self.log: + if i["id"] == identifier: + return i def get_log(self): - return self.log + with self.loglock: + return self.log def main(args): # pragma: no cover diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 134d27bc..2b60e618 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -18,7 +18,7 @@ class WebsocketsProtocol: frm = websockets.Frame.from_file(self.pathod_handler.rfile) except NetlibException as e: lg("Error reading websocket frame: %s" % e) - break + return None, None ended = time.time() lg(frm.human_readable()) retlog = dict( diff --git a/pathod/test.py b/pathod/test.py index 23b7a5b6..11462729 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -1,12 +1,14 @@ from six.moves import cStringIO as StringIO import threading +import time + from six.moves import queue -import requests -import requests.packages.urllib3 from . import pathod -requests.packages.urllib3.disable_warnings() + +class TimeoutError(Exception): + pass class Daemon: @@ -39,39 +41,51 @@ class Daemon: """ return "%s/p/%s" % (self.urlbase, spec) - def info(self): - """ - Return some basic info about the remote daemon. - """ - resp = requests.get("%s/api/info" % self.urlbase, verify=False) - return resp.json() - def text_log(self): return self.logfp.getvalue() + def wait_for_silence(self, timeout=5): + start = time.time() + while 1: + if time.time() - start >= timeout: + raise TimeoutError( + "%s service threads still alive" % + self.thread.server.handler_counter.count + ) + if self.thread.server.handler_counter.count == 0: + return + + def expect_log(self, n, timeout=5): + l = [] + start = time.time() + while True: + l = self.log() + if time.time() - start >= timeout: + return None + if len(l) >= n: + break + return l + def last_log(self): """ Returns the last logged request, or None. """ - l = self.log() + l = self.expect_log(1) if not l: return None - return l[0] + return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ - resp = requests.get("%s/api/log" % self.urlbase, verify=False) - return resp.json()["log"] + return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ - self.logfp.truncate(0) - resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) - return resp.ok + return self.thread.server.clear_log() def shutdown(self): """ @@ -88,6 +102,7 @@ class _PaThread(threading.Thread): self.name = "PathodThread" self.iface, self.q, self.ssl = iface, q, ssl self.daemonargs = daemonargs + self.server = None def run(self): self.server = pathod.Pathod( @@ -11,8 +11,7 @@ addopts = --capture=no [coverage:run] branch = True -include = *mitmproxy*, *netlib*, *pathod* -omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py +omit = *contrib*, *tnetstring*, *platform*, *main.py [coverage:report] show_missing = True diff --git a/test/pathod/test_app.py b/test/pathod/test_app.py index ac89c44c..fbaa773c 100644 --- a/test/pathod/test_app.py +++ b/test/pathod/test_app.py @@ -11,11 +11,11 @@ class TestApp(tutils.DaemonTests): def test_about(self): r = self.getpath("/about") - assert r.ok + assert r.status_code == 200 def test_download(self): r = self.getpath("/download") - assert r.ok + assert r.status_code == 200 def test_docs(self): assert self.getpath("/docs/pathod").status_code == 200 @@ -27,7 +27,7 @@ class TestApp(tutils.DaemonTests): def test_log(self): assert self.getpath("/log").status_code == 200 assert self.get("200:da").status_code == 200 - id = self.d.log()[0]["id"] + id = self.d.expect_log(1)[0]["id"] assert self.getpath("/log").status_code == 200 assert self.getpath("/log/%s" % id).status_code == 200 assert self.getpath("/log/9999999").status_code == 404 diff --git a/test/pathod/test_language_generators.py b/test/pathod/test_language_generators.py index 0fceae85..51f55991 100644 --- a/test/pathod/test_language_generators.py +++ b/test/pathod/test_language_generators.py @@ -7,24 +7,27 @@ import tutils def test_randomgenerator(): g = generators.RandomGenerator("bytes", 100) assert repr(g) + assert g[0] + assert len(g[0]) == 1 assert len(g[:10]) == 10 assert len(g[1:10]) == 9 assert len(g[:1000]) == 100 assert len(g[1000:1001]) == 0 - assert g[0] def test_filegenerator(): with tutils.tmpdir() as t: path = os.path.join(t, "foo") f = open(path, "wb") - f.write("x" * 10000) + f.write(b"x" * 10000) f.close() g = generators.FileGenerator(path) assert len(g) == 10000 - assert g[0] == "x" - assert g[-1] == "x" - assert g[0:5] == "xxxxx" + assert g[0] == b"x" + assert g[-1] == b"x" + assert g[0:5] == b"xxxxx" + assert len(g[1:10]) == 9 + assert len(g[10000:10001]) == 0 assert repr(g) # remove all references to FileGenerator instance to close the file # handle. diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 4d969158..ec9c169f 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -1,7 +1,6 @@ from six.moves import cStringIO as StringIO -import pytest -from pathod import pathod, version +from pathod import pathod from netlib import tcp from netlib.exceptions import HttpException, TlsException import tutils @@ -129,7 +128,6 @@ class CommonTests(tutils.DaemonTests): assert self.d.last_log() # FIXME: Other binary data elements - @pytest.mark.skip(reason="race condition") def test_sizelimit(self): r = self.get("200:b@1g") assert r.status_code == 800 @@ -140,21 +138,15 @@ class CommonTests(tutils.DaemonTests): r, _ = self.pathoc([r"get:'/p/200':i0,'\r\n'"]) assert r[0].status_code == 200 - def test_info(self): - assert tuple(self.d.info()["version"]) == version.IVERSION - - @pytest.mark.skip(reason="race condition") def test_logs(self): - assert self.d.clear_log() - assert not self.d.last_log() + self.d.clear_log() assert self.get("202:da") - assert len(self.d.log()) == 1 - assert self.d.clear_log() + assert self.d.expect_log(1) + self.d.clear_log() assert len(self.d.log()) == 0 def test_disconnect(self): - rsp = self.get("202:b@100k:d200") - assert len(rsp.content) < 200 + tutils.raises("unexpected eof", self.get, "202:b@100k:d200") def test_parserr(self): rsp = self.get("400:msg,b:") @@ -166,7 +158,7 @@ class CommonTests(tutils.DaemonTests): assert rsp.content.strip() == "testfile" def test_anchor(self): - rsp = self.getpath("anchor/foo") + rsp = self.getpath("/anchor/foo") assert rsp.status_code == 202 def test_invalid_first_line(self): @@ -223,7 +215,6 @@ class CommonTests(tutils.DaemonTests): ) assert r[1].payload == "test" - @pytest.mark.skip(reason="race condition") def test_websocket_frame_reflect_error(self): r, _ = self.pathoc( ["ws:/p/", "wf:-mask:knone:f'wf:b@10':i13,'a'"], @@ -233,7 +224,6 @@ class CommonTests(tutils.DaemonTests): # FIXME: Race Condition? assert "Parse error" in self.d.text_log() - @pytest.mark.skip(reason="race condition") def test_websocket_frame_disconnect_error(self): self.pathoc(["ws:/p/", "wf:b@10:d3"], ws_read_limit=0) assert self.d.last_log() diff --git a/test/pathod/test_test.py b/test/pathod/test_test.py index cee286a4..6399894e 100644 --- a/test/pathod/test_test.py +++ b/test/pathod/test_test.py @@ -2,6 +2,10 @@ import logging import requests from pathod import test import tutils + +import requests.packages.urllib3 + +requests.packages.urllib3.disable_warnings() logging.disable(logging.CRITICAL) diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index f7bb22e5..b9f38d86 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -3,6 +3,7 @@ import re import shutil import requests from six.moves import cStringIO as StringIO +import urllib from netlib import tcp from netlib import utils @@ -63,10 +64,11 @@ class DaemonTests(object): shutil.rmtree(cls.confdir) def teardown(self): + self.d.wait_for_silence() if not (self.noweb or self.noapi): self.d.clear_log() - def getpath(self, path, params=None): + def _getpath(self, path, params=None): scheme = "https" if self.ssl else "http" resp = requests.get( "%s://localhost:%s/%s" % ( @@ -79,9 +81,29 @@ class DaemonTests(object): ) return resp + def getpath(self, path, params=None): + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + with c.connect(): + if params: + path = path + "?" + urllib.urlencode(params) + resp = c.request("get:%s" % path) + return resp + def get(self, spec): - resp = requests.get(self.d.p(spec), verify=False) - return resp + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + with c.connect(): + resp = c.request("get:/p/%s" % urllib.quote(spec).encode("string_escape")) + return resp def pathoc( self, @@ -106,16 +128,16 @@ class DaemonTests(object): fp=logfp, use_http2=use_http2, ) - c.connect(connect_to) - ret = [] - for i in specs: - resp = c.request(i) - if resp: - ret.append(resp) - for frm in c.wait(): - ret.append(frm) - c.stop() - return ret, logfp.getvalue() + with c.connect(connect_to): + ret = [] + for i in specs: + resp = c.request(i) + if resp: + ret.append(resp) + for frm in c.wait(): + ret.append(frm) + c.stop() + return ret, logfp.getvalue() tmpdir = tutils.tmpdir diff --git a/web/package.json b/web/package.json index b7f6ab2a..bb32a2dc 100644 --- a/web/package.json +++ b/web/package.json @@ -34,6 +34,7 @@ "babel-eslint": "^6.0.4", "babel-jest": "^12.0.2", "babel-plugin-transform-class-properties": "^6.6.0", + "babel-plugin-transform-object-rest-spread": "^6.8.0", "babel-preset-es2015": "^6.6.0", "babel-preset-react": "^6.5.0", "babelify": "^7.3.0", diff --git a/web/src/js/actions.js b/web/src/js/actions.js index 6ded4c44..c77cdf73 100644 --- a/web/src/js/actions.js +++ b/web/src/js/actions.js @@ -1,6 +1,6 @@ import $ from "jquery"; -import _ from "lodash"; import {AppDispatcher} from "./dispatcher.js"; +import {fetchApi} from "./utils.js"; export var ActionTypes = { // Connection @@ -117,6 +117,16 @@ export var FlowActions = { }, clear: function(){ $.post("/clear"); + }, + download: () => window.location = "/flows/dump", + + upload: (file) => { + let data = new FormData(); + data.append('file', file); + fetchApi("/flows/dump", { + method: 'post', + body: data + }) } }; diff --git a/web/src/js/components/header.js b/web/src/js/components/header.js index e329b3f5..4152e95c 100644 --- a/web/src/js/components/header.js +++ b/web/src/js/components/header.js @@ -331,12 +331,19 @@ var FileMenu = React.createClass({ } }, handleOpenClick: function (e) { + this.fileInput.click(); + e.preventDefault(); + }, + handleOpenFile: function (e) { + if (e.target.files.length > 0) { + FlowActions.upload(e.target.files[0]); + this.fileInput.value = ""; + } e.preventDefault(); - console.error("unimplemented: handleOpenClick"); }, handleSaveClick: function (e) { e.preventDefault(); - console.error("unimplemented: handleSaveClick"); + FlowActions.download(); }, handleShutdownClick: function (e) { e.preventDefault(); @@ -355,6 +362,20 @@ var FileMenu = React.createClass({ New </a> </li> + <li> + <a href="#" onClick={this.handleOpenClick}> + <i className="fa fa-fw fa-folder-open"></i> + Open... + </a> + <input ref={(ref) => this.fileInput = ref} className="hidden" type="file" onChange={this.handleOpenFile}/> + + </li> + <li> + <a href="#" onClick={this.handleSaveClick}> + <i className="fa fa-fw fa-floppy-o"></i> + Save... + </a> + </li> <li role="presentation" className="divider"></li> <li> <a href="http://mitm.it/" target="_blank"> @@ -363,18 +384,6 @@ var FileMenu = React.createClass({ </a> </li> {/* - <li> - <a href="#" onClick={this.handleOpenClick}> - <i className="fa fa-fw fa-folder-open"></i> - Open - </a> - </li> - <li> - <a href="#" onClick={this.handleSaveClick}> - <i className="fa fa-fw fa-save"></i> - Save - </a> - </li> <li role="presentation" className="divider"></li> <li> <a href="#" onClick={this.handleShutdownClick}> diff --git a/web/src/js/store/store.js b/web/src/js/store/store.js index e41b2eef..a16a0369 100644 --- a/web/src/js/store/store.js +++ b/web/src/js/store/store.js @@ -2,7 +2,7 @@ import _ from "lodash"; import $ from "jquery"; import {EventEmitter} from 'events'; - +import { EventLogActions } from "../actions.js" import {ActionTypes, StoreCmds} from "../actions.js"; import {AppDispatcher} from "../dispatcher.js"; diff --git a/web/src/js/utils.js b/web/src/js/utils.js index 2627cf58..97737b20 100644 --- a/web/src/js/utils.js +++ b/web/src/js/utils.js @@ -80,7 +80,7 @@ function getCookie(name) { var r = document.cookie.match(new RegExp("\\b" + name + "=([^;]*)\\b")); return r ? r[1] : undefined; } -var xsrf = $.param({_xsrf: getCookie("_xsrf")}); +const xsrf = `_xsrf=${getCookie("_xsrf")}`; //Tornado XSRF Protection. $.ajaxPrefilter(function (options) { @@ -101,4 +101,16 @@ $(document).ajaxError(function (event, jqXHR, ajaxSettings, thrownError) { console.error(thrownError, message, arguments); actions.EventLogActions.add_event(thrownError + ": " + message); alert(message); -});
\ No newline at end of file +}); + +export function fetchApi(url, options) { + if(url.indexOf("?") === -1){ + url += "?" + xsrf; + } else { + url += "&" + xsrf; + } + return fetch(url, { + ...options, + credentials: 'same-origin' + }); +}
\ No newline at end of file |