diff options
| -rw-r--r-- | libpathod/language/__init__.py | 14 | ||||
| -rw-r--r-- | libpathod/language/http.py | 6 | ||||
| -rw-r--r-- | libpathod/language/http2.py | 113 | ||||
| -rw-r--r-- | libpathod/pathoc.py | 8 | ||||
| -rw-r--r-- | libpathod/pathoc_cmdline.py | 15 | ||||
| -rw-r--r-- | libpathod/pathod.py | 214 | ||||
| -rw-r--r-- | libpathod/pathod_cmdline.py | 15 | ||||
| -rw-r--r-- | libpathod/utils.py | 10 | ||||
| -rw-r--r-- | test/test_language_http2.py | 3 | ||||
| -rw-r--r-- | test/test_pathoc.py | 7 |
10 files changed, 296 insertions, 109 deletions
diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py index ae9a8c76..10050bf8 100644 --- a/libpathod/language/__init__.py +++ b/libpathod/language/__init__.py @@ -19,7 +19,7 @@ def expand(msg): yield msg -def parse_pathod(s): +def parse_pathod(s, use_http2=False): """ May raise ParseException """ @@ -28,12 +28,17 @@ def parse_pathod(s): except UnicodeError: raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) try: - reqs = pp.Or( - [ + if use_http2: + expressions = [ + # http2.Frame.expr(), + http2.Response.expr(), + ] + else: + expressions = [ websockets.WebsocketFrame.expr(), http.Response.expr(), ] - ).parseString(s, parseAll=True) + reqs = pp.Or(expressions).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) return itertools.chain(*[expand(i) for i in reqs]) @@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False): websockets.WebsocketClientFrame.expr(), http.Request.expr(), ] - reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) diff --git a/libpathod/language/http.py b/libpathod/language/http.py index 9a8404f0..115f8069 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -367,10 +367,6 @@ class Request(_HTTPMessage): return ":".join([i.spec() for i in self.tokens]) -class PathodErrorResponse(Response): - pass - - def make_error_response(reason, body=None): tokens = [ Code("800"), @@ -381,4 +377,4 @@ def make_error_response(reason, body=None): Reason(base.TokValueLiteral(reason)), Body(base.TokValueLiteral("pathod error: " + (body or reason))), ] - return PathodErrorResponse(tokens) + return Response(tokens) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index 4a5b9084..1d2517d3 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -45,14 +45,20 @@ class Body(base.Value): class Times(base.Integer): preamble = "x" +class Code(base.Integer): + pass class Request(message.Message): comps = ( Header, Body, - Times, ) + logattrs = ["method", "path"] + + def __init__(self, tokens): + super(Response, self).__init__(tokens) + self.rendered_values = None @property def method(self): @@ -87,7 +93,6 @@ class Request(message.Message): Method.expr(), base.Sep, Path.expr(), - base.Sep, pp.ZeroOrMore(base.Sep + atom) ] ) @@ -95,21 +100,109 @@ class Request(message.Message): return resp def resolve(self, settings, msg=None): - tokens = self.tokens[:] - return self.__class__( - [i.resolve(settings, self) for i in tokens] + return self + + def values(self, settings): + if self.rendered_values: + return self.rendered_values + else: + headers = self.headers + if headers: + headers = headers.values(settings) + + body = self.body + if body: + body = body.string() + + self.rendered_values = settings.protocol.create_request( + self.method.string(), + self.path.string(), + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values + + def spec(self): + return ":".join([i.spec() for i in self.tokens]) + + +class Response(message.Message): + unique_name = None + comps = ( + Header, + Body, + ) + + def __init__(self, tokens): + super(Response, self).__init__(tokens) + self.rendered_values = None + self.stream_id = 0 + + @property + def code(self): + return self.tok(Code) + + @property + def headers(self): + return self.toks(Header) + + @property + def body(self): + return self.tok(Body) + + @property + def actions(self): + return [] + + def resolve(self, settings, msg=None): + return self + + @classmethod + def expr(klass): + parts = [i.expr() for i in klass.comps] + atom = pp.MatchFirst(parts) + resp = pp.And( + [ + Code.expr(), + pp.ZeroOrMore(base.Sep + atom) + ] ) + resp = resp.setParseAction(klass) + return resp def values(self, settings): - return settings.protocol.create_request( - self.method.string(), - self.path, - self.headers, - self.body) + if self.rendered_values: + return self.rendered_values + else: + headers = self.headers + if headers: + headers = headers.values(settings) + + body = self.body + if body: + body = body.values(settings) + + self.rendered_values = settings.protocol.create_response( + self.code.string(), + self.stream_id, + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values def spec(self): return ":".join([i.spec() for i in self.tokens]) +def make_error_response(reason, body=None): + raise NotImplementedError + # tokens = [ + # Code("800"), + # Header( + # base.TokValueLiteral("Content-Type"), + # base.TokValueLiteral("text/plain") + # ), + # Reason(base.TokValueLiteral(reason)), + # Body(base.TokValueLiteral("pathod error: " + (body or reason))), + # ] + # return Response(tokens) # class Frame(message.Message): # pass diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index e9bd5f56..9c021360 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -155,13 +155,14 @@ class Pathoc(tcp.TCPClient): # SSL ssl=None, sni=None, - sslversion=4, + sslversion='SSLv23', clientcert=None, ciphers=None, # HTTP/2 use_http2=False, http2_skip_connection_preface=False, + http2_framedump = False, # Websockets ws_read_limit = None, @@ -199,6 +200,7 @@ class Pathoc(tcp.TCPClient): self.use_http2 = use_http2 self.http2_skip_connection_preface = http2_skip_connection_preface + self.http2_framedump = http2_framedump self.ws_read_limit = ws_read_limit @@ -219,7 +221,6 @@ class Pathoc(tcp.TCPClient): if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2." print >> sys.stderr, "Pathoc might not be working as expected without ALPN." - self.protocol = http2.HTTP2Protocol(self) else: # TODO: create HTTP or Websockets protocol @@ -298,7 +299,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2: self.protocol.check_alpn() if not self.http2_skip_connection_preface: - self.protocol.perform_connection_preface() + self.protocol.perform_client_connection_preface() if self.timeout: self.settimeout(self.timeout) @@ -466,6 +467,7 @@ def main(args): # pragma: nocover ciphers = args.ciphers, use_http2 = args.use_http2, http2_skip_connection_preface = args.http2_skip_connection_preface, + http2_framedump = args.http2_framedump, showreq = args.showreq, showresp = args.showresp, explain = args.explain, diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py index dcd75d11..1d0df3b5 100644 --- a/libpathod/pathoc_cmdline.py +++ b/libpathod/pathoc_cmdline.py @@ -109,12 +109,11 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): help="SSL cipher specification" ) group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""" - Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23. - """ + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" ) group = parser.add_argument_group( @@ -156,6 +155,10 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): "-x", dest="hexdump", action="store_true", default=False, help="Output in hexdump format" ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) args = parser.parse_args(argv[1:]) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index f1496181..535340cb 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -7,7 +7,7 @@ import urllib import re import time -from netlib import tcp, http, wsgi, certutils, websockets +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict from . import version, app, language, utils, log import language.http @@ -40,7 +40,7 @@ class SSLOptions: sslversion=tcp.SSLv23_METHOD, ciphers=None, certs=None, - alpn_select=None, + alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2, ): self.confdir = confdir self.cn = cn @@ -69,32 +69,37 @@ class PathodHandler(tcp.BaseHandler): wbufsize = 0 sni = None - def __init__(self, connection, address, server, logfp, settings): - self.logfp = logfp + def __init__(self, connection, address, server, logfp, settings, http2_framedump=False): tcp.BaseHandler.__init__(self, connection, address, server) + self.logfp = logfp self.settings = copy.copy(settings) + self.protocol = None + self.use_http2 = False + self.http2_framedump = http2_framedump - def handle_sni(self, connection): + def _handle_sni(self, connection): self.sni = connection.get_servername() def http_serve_crafted(self, crafted): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + error, crafted = self.server.check_policy( crafted, self.settings ) if error: - err = language.http.make_error_response(error) + err = self.make_http_error_response(error) language.serve(err, self.wfile, self.settings) return None, dict( type="error", msg = error ) - if self.server.explain and not isinstance( - crafted, - language.http.PathodErrorResponse - ): + if self.server.explain and not hasattr(crafted, 'is_error_response'): crafted = crafted.freeze(self.settings) log.write(self.logfp, ">> Spec: %s" % crafted.spec()) + response_log = language.serve( crafted, self.wfile, @@ -154,6 +159,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_connect(self, connect, lg): """ + This method is HTTP/1 only. + Handle a CONNECT request. """ http.read_headers(self.rfile) @@ -171,7 +178,7 @@ class PathodHandler(tcp.BaseHandler): self.convert_to_ssl( cert, key, - handle_sni=self.handle_sni, + handle_sni=self._handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, @@ -185,10 +192,12 @@ class PathodHandler(tcp.BaseHandler): def handle_http_app(self, method, path, headers, content, lg): """ + This method is HTTP/1 only. + Handle a request to the built-in app. """ if self.server.noweb: - crafted = language.http.make_error_response("Access Denied") + crafted = self.make_http_error_response("Access Denied") language.serve(crafted, self.wfile, self.settings) return None, dict( type="error", @@ -209,6 +218,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_request(self): """ + This method is HTTP/1 and HTTP/2 capable. + Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect @@ -217,28 +228,24 @@ class PathodHandler(tcp.BaseHandler): lr = self.rfile if self.server.logreq else None lw = self.wfile if self.server.logresp else None with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg: - line = http.get_request_line(self.rfile) - if not line: - # Normal termination - return None, None - - m = utils.MemBool() - if m(http.parse_init_connect(line)): - return self.handle_http_connect(m.v, lg) - elif m(http.parse_init_proxy(line)): - method, _, _, _, path, httpversion = m.v - elif m(http.parse_init_http(line)): - method, path, httpversion = m.v + if self.use_http2: + self.protocol.perform_server_connection_preface() + stream_id, headers, body = self.protocol.read_request() + method = headers[':method'] + path = headers[':path'] + headers = odict.ODict(headers) else: - s = "Invalid first line: %s" % repr(line) - lg(s) - return None, dict(type="error", msg=s) - - headers = http.read_headers(self.rfile) - if headers is None: - s = "Invalid headers" - lg(s) - return None, dict(type="error", msg=s) + req = self.read_http_request(lg) + if 'next_handle' in req: + return req['next_handle'] + if 'errors' in req: + return None, req['errors'] + if not 'method' in req or not 'path' in req: + return None, None + method = req['method'] + path = req['path'] + headers = req['headers'] + body = req['body'] clientcert = None if self.clientcert: @@ -258,7 +265,7 @@ class PathodHandler(tcp.BaseHandler): path=path, method=method, headers=headers.lst, - httpversion=httpversion, + # httpversion=httpversion, sni=self.sni, remote_address=self.address(), clientcert=clientcert, @@ -268,16 +275,6 @@ class PathodHandler(tcp.BaseHandler): if self.ssl_established: retlog["cipher"] = self.get_current_cipher() - try: - content = http.read_http_body( - self.rfile, headers, None, - method, None, True - ) - except http.HttpError as s: - s = str(s) - lg(s) - return None, dict(type="error", msg=s) - m = utils.MemBool() websocket_key = websockets.check_client_handshake(headers) self.settings.websocket_key = websocket_key @@ -288,28 +285,37 @@ class PathodHandler(tcp.BaseHandler): anchor_gen = language.parse_pathod("ws") else: anchor_gen = None - for i in self.server.anchors: - if i[0].match(path): - anchor_gen = i[1] + + for regex, spec in self.server.anchors: + if regex.match(path): + anchor_gen = language.parse_pathod(spec, self.use_http2) break else: - print(self.server.craftanchor) if m(path.startswith(self.server.craftanchor)): spec = urllib.unquote(path)[len(self.server.craftanchor):] if spec: try: - anchor_gen = language.parse_pathod(spec) + anchor_gen = language.parse_pathod(spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) - anchor_gen = iter([language.http.make_error_response( + anchor_gen = iter([self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % ( v.msg + v.marked() ) )]) + else: + if self.use_http2: + raise NotImplementedError(\ + "HTTP/2 only supports request/response with the craft anchor point.") + if anchor_gen: spec = anchor_gen.next() + + if self.use_http2 and isinstance(spec, language.http2.Response): + spec.stream_id = stream_id + lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( spec @@ -319,28 +325,77 @@ class PathodHandler(tcp.BaseHandler): else: return nexthandler, retlog else: - return self.handle_http_app(method, path, headers, content, lg) + return self.handle_http_app(method, path, headers, body, lg) - def addlog(self, log): - # FIXME: The bytes in the log should not be escaped. We do this at the - # moment because JSON encoding can't handle binary data, and I don't - # want to base64 everything. - if self.server.logreq: - bytes = self.rfile.get_log().encode("string_escape") - log["request_bytes"] = bytes - if self.server.logresp: - bytes = self.wfile.get_log().encode("string_escape") - log["response_bytes"] = bytes - self.server.add_log(log) + def read_http_request(self, lg): + """ + This method is HTTP/1 only. + """ + line = http.get_request_line(self.rfile) + if not line: + # Normal termination + return dict() + + m = utils.MemBool() + if m(http.parse_init_connect(line)): + return dict(next_handle=self.handle_http_connect(m.v, lg)) + elif m(http.parse_init_proxy(line)): + method, _, _, _, path, httpversion = m.v + elif m(http.parse_init_http(line)): + method, path, httpversion = m.v + else: + s = "Invalid first line: %s" % repr(line) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + headers = http.read_headers(self.rfile) + if headers is None: + s = "Invalid headers" + lg(s) + return dict(errors=dict(type="error", msg=s)) + + try: + body = http.read_http_body( + self.rfile, + headers, + None, + method, + None, + True, + ) + except http.HttpError as s: + s = str(s) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + return dict( + method=method, + path=path, + headers=headers, + body=body) + + def make_http_error_response(self, reason, body=None): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + if self.use_http2: + resp = language.http2.make_error_response(reason, body) + else: + resp = language.http.make_error_response(reason, body) + resp.is_error_response = True + return resp def handle(self): + self.settimeout(self.server.timeout) + if self.server.ssl: try: cert, key, _ = self.server.ssloptions.get_cert(None) self.convert_to_ssl( cert, key, - handle_sni=self.handle_sni, + dhparams=self.server.ssloptions.certstore.dhparams, + handle_sni=self._handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, @@ -356,8 +411,20 @@ class PathodHandler(tcp.BaseHandler): ) log.write(self.logfp, s) return - self.settimeout(self.server.timeout) + + alp = self.get_alpn_proto_negotiated() + if alp == http2.HTTP2Protocol.ALPN_PROTO_H2: + self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump) + self.use_http2 = True + + # if not self.protocol: + # # TODO: create HTTP or Websockets protocol + # self.protocol = None + + self.settings.protocol = self.protocol + handler = self.handle_http_request + while not self.finished: handler, l = handler() if l: @@ -365,6 +432,18 @@ class PathodHandler(tcp.BaseHandler): if not handler: return + def addlog(self, log): + # FIXME: The bytes in the log should not be escaped. We do this at the + # moment because JSON encoding can't handle binary data, and I don't + # want to base64 everything. + if self.server.logreq: + bytes = self.rfile.get_log().encode("string_escape") + log["request_bytes"] = bytes + if self.server.logresp: + bytes = self.wfile.get_log().encode("string_escape") + log["response_bytes"] = bytes + self.server.add_log(log) + class Pathod(tcp.TCPServer): LOGBUF = 500 @@ -387,6 +466,7 @@ class Pathod(tcp.TCPServer): logresp=False, explain=False, hexdump=False, + http2_framedump=False, webdebug=False, logfp=sys.stdout, ): @@ -414,6 +494,7 @@ class Pathod(tcp.TCPServer): self.noapi, self.nohang = noapi, nohang self.timeout, self.logreq = timeout, logreq self.logresp, self.hexdump = logresp, hexdump + self.http2_framedump = http2_framedump self.explain = explain self.logfp = logfp @@ -451,7 +532,8 @@ class Pathod(tcp.TCPServer): client_address, self, self.logfp, - self.settings + self.settings, + self.http2_framedump, ) try: h.handle() @@ -508,7 +590,6 @@ def main(args): # pragma: nocover sslversion = utils.SSLVERSIONS[args.sslversion], certs = args.ssl_certs, sans = args.sans, - alpn_select = args.alpn_select, ) root = logging.getLogger() @@ -548,6 +629,7 @@ def main(args): # pragma: nocover logreq = args.logreq, logresp = args.logresp, hexdump = args.hexdump, + http2_framedump = args.http2_framedump, explain = args.explain, webdebug = args.webdebug ) diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py index f1bb6982..4343401f 100644 --- a/libpathod/pathod_cmdline.py +++ b/libpathod/pathod_cmdline.py @@ -139,10 +139,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): """ ) group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23.""" + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" ) group = parser.add_argument_group( @@ -172,6 +173,12 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): "-x", dest="hexdump", action="store_true", default=False, help="Log request/response in hexdump format" ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) + + args = parser.parse_args(argv[1:]) certs = [] diff --git a/libpathod/utils.py b/libpathod/utils.py index 9bd2812e..481c5137 100644 --- a/libpathod/utils.py +++ b/libpathod/utils.py @@ -3,10 +3,12 @@ import sys from netlib import tcp SSLVERSIONS = { - 1: tcp.TLSv1_METHOD, - 2: tcp.SSLv2_METHOD, - 3: tcp.SSLv3_METHOD, - 4: tcp.SSLv23_METHOD, + 'TLSv1.2': tcp.TLSv1_2_METHOD, + 'TLSv1.1': tcp.TLSv1_1_METHOD, + 'TLSv1': tcp.TLSv1_METHOD, + 'SSLv3': tcp.SSLv3_METHOD, + 'SSLv2': tcp.SSLv2_METHOD, + 'SSLv23': tcp.SSLv23_METHOD, } SIZE_UNITS = dict( diff --git a/test/test_language_http2.py b/test/test_language_http2.py index de3e5cf9..3c751fd1 100644 --- a/test/test_language_http2.py +++ b/test/test_language_http2.py @@ -1,5 +1,6 @@ import cStringIO +from netlib import tcp from libpathod import language from libpathod.language import http2, base import netlib @@ -64,7 +65,7 @@ class TestRequest: s, language.Settings( request_host = "foo.com", - protocol = netlib.http2.HTTP2Protocol(None) + protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234))) ) ) diff --git a/test/test_pathoc.py b/test/test_pathoc.py index e1e1fe97..d39f9275 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -230,9 +230,6 @@ class TestDaemon(_TestDaemon): class TestDaemonHTTP2(_TestDaemon): ssl = True - ssloptions = pathod.SSLOptions( - alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2, - ) def test_http2(self): c = pathoc.Pathoc( @@ -270,5 +267,5 @@ class TestDaemonHTTP2(_TestDaemon): use_http2 = True, ) c.connect() - resp = c.request("get:/api/info") - assert tuple(json.loads(resp.content)["version"]) == version.IVERSION + resp = c.request("get:/p/200") + assert resp.status_code == "200" |