diff options
Diffstat (limited to 'pathod')
| -rw-r--r-- | pathod/app.py | 9 | ||||
| -rw-r--r-- | pathod/language/__init__.py | 15 | ||||
| -rw-r--r-- | pathod/language/base.py | 25 | ||||
| -rw-r--r-- | pathod/language/exceptions.py | 1 | ||||
| -rw-r--r-- | pathod/language/generators.py | 57 | ||||
| -rw-r--r-- | pathod/language/http.py | 1 | ||||
| -rw-r--r-- | pathod/language/http2.py | 7 | ||||
| -rw-r--r-- | pathod/language/websockets.py | 1 | ||||
| -rw-r--r-- | pathod/language/writer.py | 1 | ||||
| -rw-r--r-- | pathod/log.py | 12 | ||||
| -rw-r--r-- | pathod/pathoc.py | 19 | ||||
| -rw-r--r-- | pathod/pathod.py | 25 | ||||
| -rw-r--r-- | pathod/pathod_cmdline.py | 15 | ||||
| -rw-r--r-- | pathod/protocols/__init__.py | 6 | ||||
| -rw-r--r-- | pathod/protocols/http.py | 6 | ||||
| -rw-r--r-- | pathod/protocols/http2.py | 3 | ||||
| -rw-r--r-- | pathod/protocols/websockets.py | 2 | ||||
| -rw-r--r-- | pathod/test.py | 49 | ||||
| -rw-r--r-- | pathod/utils.py | 50 | ||||
| -rw-r--r-- | pathod/version.py | 7 |
20 files changed, 154 insertions, 157 deletions
diff --git a/pathod/app.py b/pathod/app.py index aa00ed69..e3216c58 100644 --- a/pathod/app.py +++ b/pathod/app.py @@ -1,10 +1,11 @@ import logging import pprint -from six.moves import cStringIO as StringIO +import io import copy from flask import Flask, jsonify, render_template, request, abort, make_response -from . import version, language, utils +from . import version, language from netlib.http import user_agents +from netlib import strutils logging.basicConfig(level="DEBUG") EXAMPLE_HOST = "example.com" @@ -145,7 +146,7 @@ def make_app(noapi, debug): args["marked"] = v.marked() return render(template, False, **args) - s = StringIO() + s = io.BytesIO() settings = copy.copy(app.config["pathod"].settings) settings.request_host = EXAMPLE_HOST @@ -166,7 +167,7 @@ def make_app(noapi, debug): settings.websocket_key = EXAMPLE_WEBSOCKET_KEY language.serve(safe, s, settings) - args["output"] = utils.escape_unprintables(s.getvalue()) + args["output"] = strutils.bytes_to_escaped_str(s.getvalue()) return render(template, False, **args) @app.route('/response_preview') diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py index 32199e08..0841196e 100644 --- a/pathod/language/__init__.py +++ b/pathod/language/__init__.py @@ -1,19 +1,26 @@ +from __future__ import absolute_import + import itertools import time +from six.moves import range import pyparsing as pp from . import http, http2, websockets, writer, exceptions -from exceptions import * -from base import Settings -assert Settings # prevent pyflakes from messing with this +from .exceptions import RenderError, FileAccessDenied, ParseException +from .base import Settings + +__all__ = [ + "RenderError", "FileAccessDenied", "ParseException", + "Settings", +] def expand(msg): times = getattr(msg, "times", None) if times: - for j_ in xrange(int(times.value)): + for j_ in range(int(times.value)): yield msg.strike_token("times") else: yield msg diff --git a/pathod/language/base.py b/pathod/language/base.py index a4302998..11ee0623 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -3,9 +3,14 @@ import os import abc import pyparsing as pp -from .. import utils +import six +from six.moves import reduce +from netlib import strutils +from netlib import human + from . import generators, exceptions + class Settings(object): def __init__( @@ -105,7 +110,7 @@ class Token(object): class _TokValueLiteral(Token): def __init__(self, val): - self.val = val.decode("string_escape") + self.val = strutils.escaped_str_to_bytes(val) def get_generator(self, settings_): return self.val @@ -130,7 +135,7 @@ class TokValueLiteral(_TokValueLiteral): return v def spec(self): - inner = self.val.encode("string_escape") + inner = strutils.bytes_to_escaped_str(self.val) inner = inner.replace(r"\'", r"\x27") return "'" + inner + "'" @@ -143,7 +148,7 @@ class TokValueNakedLiteral(_TokValueLiteral): return e.setParseAction(lambda x: cls(*x)) def spec(self): - return self.val.encode("string_escape") + return strutils.bytes_to_escaped_str(self.val) class TokValueGenerate(Token): @@ -154,14 +159,14 @@ class TokValueGenerate(Token): self.usize, self.unit, self.datatype = usize, unit, datatype def bytes(self): - return self.usize * utils.SIZE_UNITS[self.unit] + return self.usize * human.SIZE_UNITS[self.unit] def get_generator(self, settings_): return generators.RandomGenerator(self.datatype, self.bytes()) def freeze(self, settings): g = self.get_generator(settings) - return TokValueLiteral(g[:].encode("string_escape")) + return TokValueLiteral(strutils.bytes_to_escaped_str(g[:])) @classmethod def expr(cls): @@ -169,7 +174,7 @@ class TokValueGenerate(Token): u = reduce( operator.or_, - [pp.Literal(i) for i in utils.SIZE_UNITS.keys()] + [pp.Literal(i) for i in human.SIZE_UNITS.keys()] ).leaveWhitespace() e = e + pp.Optional(u, default=None) @@ -221,7 +226,7 @@ class TokValueFile(Token): return generators.FileGenerator(s) def spec(self): - return "<'%s'" % self.path.encode("string_escape") + return "<'%s'" % strutils.bytes_to_escaped_str(self.path) TokValue = pp.MatchFirst( @@ -337,7 +342,7 @@ class OptionsOrValue(_Component): # it to be canonical. The user can specify a different case by using a # string value literal. self.option_used = False - if isinstance(value, basestring): + if isinstance(value, six.string_types): for i in self.options: # Find the exact option value in a case-insensitive way if i.lower() == value.lower(): @@ -573,4 +578,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(f.encode("string_escape"))) + return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f))) diff --git a/pathod/language/exceptions.py b/pathod/language/exceptions.py index 84ad3c02..eb22bed2 100644 --- a/pathod/language/exceptions.py +++ b/pathod/language/exceptions.py @@ -1,4 +1,3 @@ - class RenderError(Exception): pass diff --git a/pathod/language/generators.py b/pathod/language/generators.py index a17e7052..9fff3082 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -2,17 +2,19 @@ import string import random import mmap +import six + DATATYPES = dict( - ascii_letters=string.ascii_letters, - ascii_lowercase=string.ascii_lowercase, - ascii_uppercase=string.ascii_uppercase, - digits=string.digits, - hexdigits=string.hexdigits, - octdigits=string.octdigits, - punctuation=string.punctuation, - whitespace=string.whitespace, - ascii=string.printable, - bytes="".join(chr(i) for i in range(256)) + ascii_letters=string.ascii_letters.encode(), + ascii_lowercase=string.ascii_lowercase.encode(), + ascii_uppercase=string.ascii_uppercase.encode(), + digits=string.digits.encode(), + hexdigits=string.hexdigits.encode(), + octdigits=string.octdigits.encode(), + punctuation=string.punctuation.encode(), + whitespace=string.whitespace.encode(), + ascii=string.printable.encode(), + bytes=bytes(bytearray(range(256))) ) @@ -35,16 +37,25 @@ class TransformGenerator(object): def __getitem__(self, x): d = self.gen.__getitem__(x) + if isinstance(x, slice): + return self.transform(x.start, d) return self.transform(x, d) - def __getslice__(self, a, b): - d = self.gen.__getslice__(a, b) - return self.transform(a, d) - def __repr__(self): return "'transform(%s)'" % self.gen +def rand_byte(chars): + """ + Return a random character as byte from a charset. + """ + # bytearray has consistent behaviour on both Python 2 and 3 + # while bytes does not + if six.PY2: + return random.choice(chars) + return bytes([random.choice(chars)]) + + class RandomGenerator(object): def __init__(self, dtype, length): @@ -55,12 +66,10 @@ class RandomGenerator(object): return self.length def __getitem__(self, x): - return random.choice(DATATYPES[self.dtype]) - - def __getslice__(self, a, b): - b = min(b, self.length) chars = DATATYPES[self.dtype] - return "".join(random.choice(chars) for x in range(a, b)) + if isinstance(x, slice): + return b"".join(rand_byte(chars) for _ in range(*x.indices(self.length))) + return rand_byte(chars) def __repr__(self): return "%s random from %s" % (self.length, self.dtype) @@ -70,17 +79,17 @@ class FileGenerator(object): def __init__(self, path): self.path = path - self.fp = file(path, "rb") + self.fp = open(path, "rb") self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ) def __len__(self): return len(self.map) def __getitem__(self, x): - return self.map.__getitem__(x) - - def __getslice__(self, a, b): - return self.map.__getslice__(a, b) + if isinstance(x, slice): + return self.map.__getitem__(x) + # A slice of length 1 returns a byte object (not an integer) + return self.map.__getitem__(slice(x, x + 1 or self.map.size())) def __repr__(self): return "<%s" % self.path diff --git a/pathod/language/http.py b/pathod/language/http.py index a82f12fe..b2308d5e 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -11,6 +11,7 @@ from . import base, exceptions, actions, message # instead of duplicating the HTTP on-the-wire representation here. # see http2 language for an example + class WS(base.CaselessLiteral): TOK = "ws" diff --git a/pathod/language/http2.py b/pathod/language/http2.py index d5e3ca31..85d9047f 100644 --- a/pathod/language/http2.py +++ b/pathod/language/http2.py @@ -27,6 +27,7 @@ from . import base, message h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload' """ + def get_header(val, headers): """ Header keys may be Values, so we have to "generate" them as we try the @@ -48,6 +49,7 @@ class _HeaderMixin(object): self.value.get_generator(settings), ) + class _HTTP2Message(message.Message): @property def actions(self): @@ -287,13 +289,10 @@ class Request(_HTTP2Message): def spec(self): return ":".join([i.spec() for i in self.tokens]) + def make_error_response(reason, body=None): tokens = [ StatusCode("800"), Body(base.TokValueLiteral("pathod error: " + (body or reason))), ] return Response(tokens) - - -# class Frame(message.Message): -# pass diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py index 09443a95..9b752b7e 100644 --- a/pathod/language/websockets.py +++ b/pathod/language/websockets.py @@ -1,4 +1,3 @@ -import os import random import string import netlib.websockets diff --git a/pathod/language/writer.py b/pathod/language/writer.py index 1a27e1ef..22e32ce2 100644 --- a/pathod/language/writer.py +++ b/pathod/language/writer.py @@ -1,6 +1,5 @@ import time from netlib.exceptions import TcpDisconnect -import netlib.tcp BLOCKSIZE = 1024 # It's not clear what the upper limit for time.sleep is. It's lower than the diff --git a/pathod/log.py b/pathod/log.py index f203542f..5bf55de4 100644 --- a/pathod/log.py +++ b/pathod/log.py @@ -1,8 +1,8 @@ import datetime -import netlib.utils -import netlib.tcp -import netlib.http +import six + +from netlib import strutils TIMEFMT = '%d-%m-%y %H:%M:%S' @@ -53,17 +53,17 @@ class LogCtx(object): ] ) if exc_value: - raise exc_type, exc_value, traceback + six.reraise(exc_type, exc_value, traceback) def suppress(self): self.suppressed = True def dump(self, data, hexdump): if hexdump: - for line in netlib.utils.hexdump(data): + for line in strutils.hexdump(data): self("\t%s %s %s" % line) else: - for i in netlib.utils.clean_bin(data).split("\n"): + for i in strutils.clean_bin(data).split("\n"): self("\t%s" % i) def __call__(self, line): diff --git a/pathod/pathoc.py b/pathod/pathoc.py index a49ed351..5cfb4591 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -13,21 +13,24 @@ import threading import OpenSSL.crypto import six -from netlib import tcp, http, certutils, websockets, socks +from netlib import tcp, certutils, websockets, socks from netlib.exceptions import HttpException, TcpDisconnect, TcpTimeout, TlsException, TcpException, \ NetlibException from netlib.http import http1, http2 -import language.http -import language.websockets -from . import utils, log +from . import log, language import logging from netlib.tutils import treq +from netlib import strutils logging.getLogger("hpack").setLevel(logging.WARNING) +def xrepr(s): + return repr(s)[1:-1] + + class PathocError(Exception): pass @@ -43,7 +46,7 @@ class SSLInfo(object): "Cipher: %s, %s bit, %s" % self.cipher, "SSL certificate chain:" ] - for n,i in enumerate(self.certchain): + for n, i in enumerate(self.certchain): parts.append(" Certificate [%s]" % n) parts.append("\tSubject: ") for cn in i.get_subject().get_components(): @@ -74,7 +77,6 @@ class SSLInfo(object): return "\n".join(parts) - class WebsocketFrameReader(threading.Thread): def __init__( @@ -284,7 +286,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2 and not self.ssl: raise NotImplementedError("HTTP2 without SSL is not supported.") - tcp.TCPClient.connect(self) + ret = tcp.TCPClient.connect(self) if connect_to: self.http_connect(connect_to) @@ -322,6 +324,7 @@ class Pathoc(tcp.TCPClient): if self.timeout: self.settimeout(self.timeout) + return ret def stop(self): if self.ws_framereader: @@ -426,7 +429,7 @@ class Pathoc(tcp.TCPClient): finally: if resp: lg("<< %s %s: %s bytes" % ( - resp.status_code, utils.xrepr(resp.reason), len(resp.content) + resp.status_code, strutils.bytes_to_escaped_str(resp.reason), len(resp.content) )) if resp.status_code in self.ignorecodes: lg.suppress() diff --git a/pathod/pathod.py b/pathod/pathod.py index 017ce072..0449c0c1 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -6,15 +6,11 @@ import sys import threading import urllib -from netlib import tcp, http, certutils, websockets +from netlib import tcp, certutils, websockets from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \ TlsException from . import version, app, language, utils, log, protocols -import language.http -import language.actions -import language.exceptions -import language.websockets DEFAULT_CERT_DOMAIN = "pathod.net" @@ -116,7 +112,6 @@ class PathodHandler(tcp.BaseHandler): return None, response_log return self.handle_http_request, response_log - def handle_http_request(self, logger): """ Returns a (handler, log) tuple. @@ -358,6 +353,8 @@ class Pathod(tcp.TCPServer): staticdir=self.staticdir ) + self.loglock = threading.Lock() + def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. @@ -408,8 +405,7 @@ class Pathod(tcp.TCPServer): def add_log(self, d): if not self.noapi: - lock = threading.Lock() - with lock: + with self.loglock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: @@ -418,17 +414,18 @@ class Pathod(tcp.TCPServer): return d["id"] def clear_log(self): - lock = threading.Lock() - with lock: + with self.loglock: self.log = [] def log_by_id(self, identifier): - for i in self.log: - if i["id"] == identifier: - return i + with self.loglock: + for i in self.log: + if i["id"] == identifier: + return i def get_log(self): - return self.log + with self.loglock: + return self.log def main(args): # pragma: no cover diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py index 1f972a49..a4f05faf 100644 --- a/pathod/pathod_cmdline.py +++ b/pathod/pathod_cmdline.py @@ -4,7 +4,7 @@ import os import os.path import re -from netlib import tcp +from netlib import tcp, human from . import pathod, version, utils @@ -49,12 +49,12 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): help=""" URL path specifying prefix for URL crafting commands. (%s) - """%pathod.DEFAULT_CRAFT_ANCHOR + """ % pathod.DEFAULT_CRAFT_ANCHOR ) parser.add_argument( "--confdir", - action="store", type = str, dest="confdir", default='~/.mitmproxy', - help = "Configuration directory. (~/.mitmproxy)" + action="store", type=str, dest="confdir", default='~/.mitmproxy', + help="Configuration directory. (~/.mitmproxy)" ) parser.add_argument( "-d", dest='staticdir', default=None, type=str, @@ -117,8 +117,8 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): ) group.add_argument( "--cert", dest='ssl_certs', default=[], type=str, - metavar = "SPEC", action="append", - help = """ + metavar="SPEC", action="append", + help=""" Add an SSL certificate. SPEC is of the form "[domain=]path". The domain may include a wildcard, and is equal to "*" if not specified. The file at path is a certificate in PEM format. If a private key is included in @@ -177,7 +177,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): help="Output all received & sent HTTP/2 frames" ) - args = parser.parse_args(argv[1:]) args.ssl_version, args.ssl_options = tcp.sslversion_choices[args.ssl_version] @@ -206,7 +205,7 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): sizelimit = None if args.sizelimit: try: - sizelimit = utils.parse_size(args.sizelimit) + sizelimit = human.parse_size(args.sizelimit) except ValueError as v: return parser.error(v) args.sizelimit = sizelimit diff --git a/pathod/protocols/__init__.py b/pathod/protocols/__init__.py index 1a8c7dab..f8f3008f 100644 --- a/pathod/protocols/__init__.py +++ b/pathod/protocols/__init__.py @@ -1 +1,7 @@ from . import http, http2, websockets + +__all__ = [ + "http", + "http2", + "websockets", +] diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py index 1f1765cb..d09b5bf2 100644 --- a/pathod/protocols/http.py +++ b/pathod/protocols/http.py @@ -1,6 +1,6 @@ -from netlib import tcp, wsgi -from netlib.exceptions import HttpReadDisconnect, TlsException -from netlib.http import http1, Request +from netlib import wsgi +from netlib.exceptions import TlsException +from netlib.http import http1 from .. import version, language diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index a098a14e..3f45ec80 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -1,5 +1,6 @@ from netlib.http import http2 -from .. import version, app, language, utils, log +from .. import language + class HTTP2Protocol: diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 134d27bc..2b60e618 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -18,7 +18,7 @@ class WebsocketsProtocol: frm = websockets.Frame.from_file(self.pathod_handler.rfile) except NetlibException as e: lg("Error reading websocket frame: %s" % e) - break + return None, None ended = time.time() lg(frm.human_readable()) retlog = dict( diff --git a/pathod/test.py b/pathod/test.py index 23b7a5b6..11462729 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -1,12 +1,14 @@ from six.moves import cStringIO as StringIO import threading +import time + from six.moves import queue -import requests -import requests.packages.urllib3 from . import pathod -requests.packages.urllib3.disable_warnings() + +class TimeoutError(Exception): + pass class Daemon: @@ -39,39 +41,51 @@ class Daemon: """ return "%s/p/%s" % (self.urlbase, spec) - def info(self): - """ - Return some basic info about the remote daemon. - """ - resp = requests.get("%s/api/info" % self.urlbase, verify=False) - return resp.json() - def text_log(self): return self.logfp.getvalue() + def wait_for_silence(self, timeout=5): + start = time.time() + while 1: + if time.time() - start >= timeout: + raise TimeoutError( + "%s service threads still alive" % + self.thread.server.handler_counter.count + ) + if self.thread.server.handler_counter.count == 0: + return + + def expect_log(self, n, timeout=5): + l = [] + start = time.time() + while True: + l = self.log() + if time.time() - start >= timeout: + return None + if len(l) >= n: + break + return l + def last_log(self): """ Returns the last logged request, or None. """ - l = self.log() + l = self.expect_log(1) if not l: return None - return l[0] + return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ - resp = requests.get("%s/api/log" % self.urlbase, verify=False) - return resp.json()["log"] + return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ - self.logfp.truncate(0) - resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) - return resp.ok + return self.thread.server.clear_log() def shutdown(self): """ @@ -88,6 +102,7 @@ class _PaThread(threading.Thread): self.name = "PathodThread" self.iface, self.q, self.ssl = iface, q, ssl self.daemonargs = daemonargs + self.server = None def run(self): self.server = pathod.Pathod( diff --git a/pathod/utils.py b/pathod/utils.py index d1e2dd00..3276198a 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -3,15 +3,6 @@ import sys import netlib.utils -SIZE_UNITS = dict( - b=1024 ** 0, - k=1024 ** 1, - m=1024 ** 2, - g=1024 ** 3, - t=1024 ** 4, -) - - class MemBool(object): """ @@ -26,20 +17,6 @@ class MemBool(object): return bool(v) -def parse_size(s): - try: - return int(s) - except ValueError: - pass - for i in SIZE_UNITS.keys(): - if s.endswith(i): - try: - return int(s[:-1]) * SIZE_UNITS[i] - except ValueError: - break - raise ValueError("Invalid size specification.") - - def parse_anchor_spec(s): """ Return a tuple, or None on error. @@ -49,33 +26,6 @@ def parse_anchor_spec(s): return tuple(s.split("=", 1)) -def xrepr(s): - return repr(s)[1:-1] - - -def inner_repr(s): - """ - Returns the inner portion of a string or unicode repr (i.e. without the - quotes) - """ - if isinstance(s, unicode): - return repr(s)[2:-1] - else: - return repr(s)[1:-1] - - -def escape_unprintables(s): - """ - Like inner_repr, but preserves line breaks. - """ - s = s.replace("\r\n", "PATHOD_MARKER_RN") - s = s.replace("\n", "PATHOD_MARKER_N") - s = inner_repr(s) - s = s.replace("PATHOD_MARKER_RN", "\n") - s = s.replace("PATHOD_MARKER_N", "\n") - return s - - data = netlib.utils.Data(__name__) diff --git a/pathod/version.py b/pathod/version.py index 2da7637d..3441be92 100644 --- a/pathod/version.py +++ b/pathod/version.py @@ -4,3 +4,10 @@ from netlib.version import VERSION, IVERSION NAME = "pathod" NAMEVERSION = NAME + " " + VERSION + +__all__ = [ + "NAME", + "NAMEVERSION", + "VERSION", + "IVERSION", +] |