diff options
84 files changed, 948 insertions, 1869 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 2e1a477f..6b3933f9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,6 +18,8 @@ install: - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\"" test_script: - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod" +after_test: + - bash <(curl -s https://codecov.io/bash) cache: - C:\Users\appveyor\AppData\Local\pip\cache deploy_script: diff --git a/.travis.yml b/.travis.yml index 5f065d53..bba9a67f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,12 @@ matrix: git: depth: 9999999 - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py" + env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py" - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py" NO_ALPN=1 + env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py" NO_ALPN=1 - python: 2.7 env: DOCS=1 - script: 'cd docs && make html' + script: 'cd docs && SPHINXOPTS="-W" make -e html' allow_failures: - python: pypy @@ -52,7 +52,7 @@ script: - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod test/$SCOPE" after_success: - - coveralls + - bash <(curl -s https://codecov.io/bash) - | if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css new file mode 100644 index 00000000..63c7cc78 --- /dev/null +++ b/docs/_static/theme_overrides.css @@ -0,0 +1,11 @@ + +/* override table width restrictions */ +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal; +} + +.wy-table-responsive { + margin-bottom: 24px; + max-width: 100%; + overflow: visible; +} diff --git a/docs/conf.py b/docs/conf.py index 01bcce1b..3ef0c63d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -153,7 +153,7 @@ html_favicon = "favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] +html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -216,4 +216,7 @@ html_favicon = "favicon.ico" #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'mitmproxydoc'
\ No newline at end of file +htmlhelp_basename = 'mitmproxydoc' + +def setup(app): + app.add_stylesheet('theme_overrides.css') diff --git a/docs/config.rst b/docs/config.rst index 634b8703..ad55baed 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -62,7 +62,7 @@ Options The options available in the config files are precisely those available as command-line flags, with the key being the option's long name. To get a -complete list of these, use the :option:`--help` option on each of the tools. Be +complete list of these, use the ``--help`` option on each of the tools. Be careful to only specify common options in the **common.conf** file - unsupported options in this file will be detected as an error on startup. @@ -72,7 +72,7 @@ Examples common.conf ^^^^^^^^^^^ -Note that :option:`--port` is an option supported by all tools. +Note that ``--port`` is an option supported by all tools. .. code-block:: none diff --git a/docs/dev/models.rst b/docs/dev/models.rst index f2ddf242..02f36f58 100644 --- a/docs/dev/models.rst +++ b/docs/dev/models.rst @@ -56,6 +56,10 @@ Datastructures :special-members: :no-undoc-members: + .. autoclass:: decoded + +.. automodule:: netlib.multidict + .. autoclass:: MultiDictView .. automethod:: get_all @@ -67,9 +71,11 @@ Datastructures .. automethod:: items .. automethod:: to_dict - .. autoclass:: decoded +.. autoclass:: mitmproxy.models.Error + :show-inheritance: -.. automodule:: mitmproxy.models +.. autoclass:: mitmproxy.models.ServerConnection :show-inheritance: - :noindex: - :members: Error, ServerConnection, ClientConnection
\ No newline at end of file + +.. autoclass:: mitmproxy.models.ClientConnection + :show-inheritance:
\ No newline at end of file diff --git a/docs/features/anticache.rst b/docs/features/anticache.rst index 65d22bab..411b284b 100644 --- a/docs/features/anticache.rst +++ b/docs/features/anticache.rst @@ -2,7 +2,7 @@ Anticache ========= -When the :option:`--anticache` option is passed to mitmproxy, it removes headers +When the ``--anticache`` option is passed to mitmproxy, it removes headers (``if-none-match`` and ``if-modified-since``) that might elicit a ``304 not modified`` response from the server. This is useful when you want to make sure you capture an HTTP exchange in its totality. It's also often used during @@ -10,6 +10,6 @@ sure you capture an HTTP exchange in its totality. It's also often used during ================== ====================== -command-line :option:`--anticache` +command-line ``--anticache`` mitmproxy shortcut :kbd:`o` then :kbd:`a` ================== ====================== diff --git a/docs/features/clientreplay.rst b/docs/features/clientreplay.rst index b0eb6792..50740bcf 100644 --- a/docs/features/clientreplay.rst +++ b/docs/features/clientreplay.rst @@ -12,7 +12,7 @@ conversation, where requests may have been made concurrently. You may want to use client-side replay in conjunction with the :ref:`anticache` option, to make sure the server responds with complete data. -================== ================= -command-line :option:`-c path` +================== =========== +command-line ``-c path`` mitmproxy shortcut :kbd:`c` -================== ================= +================== =========== diff --git a/docs/features/passthrough.rst b/docs/features/passthrough.rst index b7b5df84..ea628dad 100644 --- a/docs/features/passthrough.rst +++ b/docs/features/passthrough.rst @@ -20,10 +20,10 @@ take a look at the :ref:`responsestreaming` feature. How it works ------------ -================== ============================= -command-line :option:`--ignore regex` +================== ====================== +command-line ``--ignore regex`` mitmproxy shortcut :kbd:`o` then :kbd:`I` -================== ============================= +================== ====================== mitmproxy allows you to specify a regex which is matched against a ``host:port`` string @@ -32,7 +32,7 @@ mitmproxy allows you to specify a regex which is matched against a ``host:port`` There are two important quirks to consider: - **In transparent mode, the ignore pattern is matched against the IP and ClientHello SNI host.** While we usually infer the - hostname from the Host header if the :option:`--host` argument is passed to mitmproxy, we do not + hostname from the Host header if the ``--host`` argument is passed to mitmproxy, we do not have access to this information before the SSL handshake. If the client uses SNI however, then we treat the SNI host as an ignore target. - In regular mode, explicit HTTP requests are never ignored. [#explicithttp]_ The ignore pattern is applied on CONNECT requests, which initiate HTTPS or clear-text WebSocket connections. @@ -42,7 +42,7 @@ Tutorial If you just want to ignore one specific domain, there's usually a bulletproof method to do so: -1. Run mitmproxy or mitmdump in verbose mode (:option:`-v`) and observe the ``host:port`` +1. Run mitmproxy or mitmdump in verbose mode (``-v``) and observe the ``host:port`` information in the serverconnect messages. mitmproxy will filter on these. 2. Take the ``host:port`` string, surround it with ^ and $, escape all dots (. becomes \\.) and use this as your ignore pattern: diff --git a/docs/features/proxyauth.rst b/docs/features/proxyauth.rst index bfd32fbd..afdbb639 100644 --- a/docs/features/proxyauth.rst +++ b/docs/features/proxyauth.rst @@ -10,8 +10,8 @@ upstream servers. For now, only HTTP Basic authentication is supported. The proxy auth options are not compatible with the transparent, socks or reverse proxy mode. -================== ============================= -command-line :option:`--nonanonymous`, - :option:`--singleuser USER`, - :option:`--htpasswd PATH` -================== ============================= +================== ====================== +command-line ``--nonanonymous``, + ``--singleuser USER``, + ``--htpasswd PATH`` +================== ====================== diff --git a/docs/features/replacements.rst b/docs/features/replacements.rst index 8f760866..b4643dd2 100644 --- a/docs/features/replacements.rst +++ b/docs/features/replacements.rst @@ -54,7 +54,7 @@ So, you might start **mitmdump** as follows: This will load the replacement text from the file ``~/xss-exploit``. -Both the :option:`--replace` and :option:`--replace-from-file` flags can be passed multiple +Both the ``--replace`` and ``--replace-from-file`` flags can be passed multiple times. @@ -65,8 +65,8 @@ The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has complete usage information. -================== ============================= -command-line :option:`--replace`, - :option:`--replace-from-file` +================== ======================= +command-line ``--replace``, + ``--replace-from-file`` mitmproxy shortcut :kbd:`o` then :kbd:`R` -================== ============================= +================== ======================= diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 9dc27bf4..66b5cae0 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -19,9 +19,9 @@ On the command-line Streaming can be enabled on the command line for all response bodies exceeding a certain size. The SIZE argument understands k/m/g suffixes, e.g. 3m for 3 megabytes. -================== ============================= -command-line :option:`--stream SIZE` -================== ============================= +================== ================= +command-line ``--stream SIZE`` +================== ================= .. warning:: diff --git a/docs/features/reverseproxy.rst b/docs/features/reverseproxy.rst index 87065e73..86a5b2e4 100644 --- a/docs/features/reverseproxy.rst +++ b/docs/features/reverseproxy.rst @@ -7,9 +7,9 @@ In reverse proxy mode, mitmproxy accepts standard HTTP(S) requests and forwards them to the specified upstream server. This is in contrast to :ref:`upstreamproxy`, in which mitmproxy forwards HTTP(S) proxy requests to an upstream proxy server. -================== ===================================== -command-line :option:`-R http[s]://hostname[:port]` -================== ===================================== +================== ================================ +command-line ``-R http[s]://hostname[:port]`` +================== ================================ Here, **http[s]** signifies if the proxy should use TLS to connect to the server. mitmproxy always accepts both encrypted and unencrypted requests and transforms diff --git a/docs/features/serverreplay.rst b/docs/features/serverreplay.rst index 261a1bd6..d70b6514 100644 --- a/docs/features/serverreplay.rst +++ b/docs/features/serverreplay.rst @@ -13,7 +13,7 @@ By default, :program:`mitmproxy` excludes request headers when matching incoming requests with responses from the replay file. This works in most circumstances, and makes it possible to replay server responses in situations where request headers would naturally vary, e.g. using a different user agent. -The :option:`--rheader headername` command-line option allows you to override +The ``--rheader headername`` command-line option allows you to override this behaviour by specifying individual headers that should be included in matching. @@ -30,10 +30,10 @@ recording. So, if they were in the past at the time of recording, they will be in the past at the time of replay, and vice versa. Cookie expiry times are updated in a similar way. -You can turn off response refreshing using the :option:`--norefresh` argument, or using +You can turn off response refreshing using the ``--norefresh`` argument, or using the :kbd:`o` options shortcut within :program:`mitmproxy`. -================== ================= -command-line :option:`-S path` +================== =========== +command-line ``-S path`` mitmproxy shortcut :kbd:`S` -================== ================= +================== =========== diff --git a/docs/features/setheaders.rst b/docs/features/setheaders.rst index cbc8b6a5..aa991bf4 100644 --- a/docs/features/setheaders.rst +++ b/docs/features/setheaders.rst @@ -13,7 +13,7 @@ Example: Set the **Host** header to "example.com" for all requests. mitmdump -R http://example.com --setheader :~q:Host:example.com -================== ============================= -command-line :option:`--setheader PATTERN` +================== ======================= +command-line ``--setheader PATTERN`` mitmproxy shortcut :kbd:`o` then :kbd:`H` -================== ============================= +================== ======================= diff --git a/docs/features/socksproxy.rst b/docs/features/socksproxy.rst index 76d4cda9..e1686f45 100644 --- a/docs/features/socksproxy.rst +++ b/docs/features/socksproxy.rst @@ -5,6 +5,6 @@ SOCKS Mode In this mode, mitmproxy acts as a SOCKS5 proxy server. -================== ================= -command-line :option:`--socks` -================== ================= +================== =========== +command-line ``--socks`` +================== =========== diff --git a/docs/features/sticky.rst b/docs/features/sticky.rst index a79cbe8d..7be5f842 100644 --- a/docs/features/sticky.rst +++ b/docs/features/sticky.rst @@ -21,7 +21,7 @@ record the authentication process once, and simply replay it on startup every ti to interact with the secured resources. ================== ====================== -command-line :option:`-t FILTER` +command-line ``-t FILTER`` mitmproxy shortcut :kbd:`o` then :kbd:`t` ================== ====================== @@ -36,6 +36,6 @@ authentication through the proxy. Note that :program:`mitmproxy` doesn't (yet) s replay of HTTP Digest authentication. ================== ====================== -command-line :option:`-u FILTER` +command-line ``-u FILTER`` mitmproxy shortcut :kbd:`o` then :kbd:`A` ================== ====================== diff --git a/docs/features/tcpproxy.rst b/docs/features/tcpproxy.rst index fd0746a2..1d6fbd12 100644 --- a/docs/features/tcpproxy.rst +++ b/docs/features/tcpproxy.rst @@ -18,7 +18,7 @@ How it works ------------ ================== ====================== -command-line :option:`--tcp HOST` +command-line ``--tcp HOST`` mitmproxy shortcut :kbd:`o` then :kbd:`T` ================== ====================== diff --git a/docs/features/upstreamcerts.rst b/docs/features/upstreamcerts.rst index af2e2226..d2a82ca2 100644 --- a/docs/features/upstreamcerts.rst +++ b/docs/features/upstreamcerts.rst @@ -17,7 +17,7 @@ certs in transparent mode. Upstream cert sniffing is on by default, and can optionally be turned off. -================== ============================= -command-line :option:`--no-upstream-cert` +================== ====================== +command-line ``--no-upstream-cert`` mitmproxy shortcut :kbd:`o` then :kbd:`U` -================== ============================= +================== ====================== diff --git a/docs/features/upstreamproxy.rst b/docs/features/upstreamproxy.rst index e06833c2..a4ccf57f 100644 --- a/docs/features/upstreamproxy.rst +++ b/docs/features/upstreamproxy.rst @@ -7,6 +7,6 @@ In this mode, mitmproxy accepts proxy requests and unconditionally forwards all requests to a specified upstream proxy server. This is in contrast to :ref:`reverseproxy`, in which mitmproxy forwards ordinary HTTP requests to an upstream server. -================== =================================== -command-line :option:`-U http://hostname[:port]` -================== =================================== +================== ============================= +command-line ``-U http://hostname[:port]`` +================== ============================= diff --git a/docs/index.rst b/docs/index.rst index d3b6f434..28c0c66f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,17 @@ tutorials/gamecenter tutorials/transparent-dhcp + +.. toctree:: + :hidden: + :caption: Pathod & Pathoc + + pathod/intro + pathod/language + pathod/library + pathod/test + + .. toctree:: :hidden: :caption: Hacking @@ -80,4 +91,3 @@ * :ref:`genindex` * :ref:`modindex` - diff --git a/docs/mitmdump.rst b/docs/mitmdump.rst index d9b4a26b..369bf0eb 100644 --- a/docs/mitmdump.rst +++ b/docs/mitmdump.rst @@ -7,7 +7,7 @@ mitmdump **mitmdump** is the command-line companion to mitmproxy. It provides tcpdump-like functionality to let you view, record, and programmatically -transform HTTP traffic. See the :option:`--help` flag output for complete +transform HTTP traffic. See the ``--help`` flag output for complete documentation. @@ -28,7 +28,7 @@ Filtering saved traffic >>> mitmdump -nr infile -w outfile "~m post" -Start mitmdump without binding to the proxy port (:option:`-n`), read all flows from +Start mitmdump without binding to the proxy port (``-n``), read all flows from infile, apply the specified filter expression (only match POSTs), and write to outfile. @@ -38,8 +38,8 @@ Client replay >>> mitmdump -nc outfile -Start mitmdump without binding to the proxy port (:option:`-n`), then replay all -requests from outfile (:option:`-c filename`). Flags combine in the obvious way, so +Start mitmdump without binding to the proxy port (``-n``), then replay all +requests from outfile (``-c filename``). Flags combine in the obvious way, so you can replay requests from one file, and write the resulting flows to another: diff --git a/docs/pathod/intro.rst b/docs/pathod/intro.rst new file mode 100644 index 00000000..bf0c531f --- /dev/null +++ b/docs/pathod/intro.rst @@ -0,0 +1,307 @@ +.. _intro: + +Pathology 101 +============= + + +pathod +------ + +Pathod is a pathological HTTP daemon designed to let you craft almost any +conceivable HTTP response, including ones that creatively violate the +standards. HTTP responses are specified using a :ref:`small, terse language +<language>` which pathod shares with its evil twin :ref:`pathoc`. To start +playing with pathod, fire up the daemon: + +>>> pathod + +By default, the service listens on port 9999 of localhost, and the default +crafting anchor point is the path **/p/**. Anything after this URL prefix is +treated as a response specifier. So, hitting the following URL will generate an +HTTP 200 response with 100 bytes of random data: + + http://localhost:9999/p/200:b@100 + +See the :ref:`language documentation <language>` to get (much) fancier. The +pathod daemon also takes a range of configuration options. To view those, use +the command-line help: + +>>> pathod --help + +Mimicing a proxy +^^^^^^^^^^^^^^^^ + +Pathod automatically responds to both straight HTTP and proxy requests. For +proxy requests, the upstream host is ignored, and the path portion of the URL +is used to match anchors. This lets you test software that supports a proxy +configuration by spoofing responses from upstream servers. + +By default, we treat all proxy CONNECT requests as HTTPS traffic, serving the +response using either pathod's built-in certificates, or the cert/key pair +specified by the user. You can over-ride this behaviour if you're testing a +client that makes a non-SSL CONNECT request using the **-C** command-line +option. + +Anchors +^^^^^^^ + +Anchors provide an alternative to specifying the response in the URL. Instead, +you attach a response to a pre-configured anchor point, specified with a regex. +When a URL matching the regex is requested, the specified response is served. + +>>> pathod -a "/foo=200" + +Here, "/foo" is the regex specifying the anchor path, and the part after the "=" +is a response specifier. + + +File Access +^^^^^^^^^^^ + +There are two operators in the :ref:`language <language>` that load contents +from file - the **+** operator to load an entire request specification from +file, and the **>** value specifier. In pathod, both of these operators are +restricted to a directory specified at startup, or disabled if no directory is +specified: + +>>> pathod -d ~/staticdir" + + +Internal Error Responses +^^^^^^^^^^^^^^^^^^^^^^^^ + +Pathod uses the non-standard 800 response code to indicate internal errors, to +distinguish them from crafted responses. For example, a request to: + + http://localhost:9999/p/foo + +... will return an 800 response because "foo" is not a valid page specifier. + + + + + +.. _pathoc: + + +pathoc +------ + +Pathoc is a perverse HTTP daemon designed to let you craft almost any +conceivable HTTP request, including ones that creatively violate the standards. +HTTP requests are specified using a :ref:`small, terse language <language>`, +which pathod shares with its server-side twin pathod. To view pathoc's complete +range of options, use the command-line help: + +>>> pathoc --help + + +Getting Started +^^^^^^^^^^^^^^^ + +The basic pattern for pathoc commands is as follows: + + pathoc hostname request [request ...] + +That is, we specify the hostname to connect to, followed by one or more +requests. Lets start with a simple example:: + + > pathoc google.com get:/ + 07-06-16 12:13:43: >> 'GET':/ + << 302 Found: 261 bytes + +Here, we make a GET request to the path / on port 80 of google.com. Pathoc's +output tells us that the server responded with a 302 redirection. We can tell +pathoc to connect using SSL, in which case the default port is changed to 443 +(you can over-ride the default port with the **-p** command-line option):: + + > pathoc -s www.google.com get:/ + 07-06-16 12:14:56: >> 'GET':/ + << 302 Found: 262 bytes + + +Multiple Requests +^^^^^^^^^^^^^^^^^ + +There are two ways to tell pathoc to issue multiple requests. The first is to specify +them on the command-line, like so:: + + > pathoc google.com get:/ get:/ + 07-06-16 12:21:04: >> 'GET':/ + << 302 Found: 261 bytes + 07-06-16 12:21:04: >> 'GET':/ + << 302 Found: 261 bytes + +In this case, pathoc issues the specified requests over the same TCP connection - +so in the above example only one connection is made to google.com + +The other way to issue multiple requests is to use the **-n** flag:: + + > pathoc -n 2 google.com get:/ + 07-06-16 12:21:04: >> 'GET':/ + << 302 Found: 261 bytes + 07-06-16 12:21:04: >> 'GET':/ + << 302 Found: 261 bytes + +The output is identical, but two separate TCP connections are made to the +upstream server. These two specification styles can be combined:: + + pathoc -n 2 google.com get:/ get:/ + + +Here, two distinct TCP connections are made, with two requests issued over +each. + + + +Basic Fuzzing +^^^^^^^^^^^^^ + +The combination of pathoc's powerful request specification language and a few +of its command-line options makes for quite a powerful basic fuzzer. Here's an +example:: + + pathoc -e -I 200 -t 2 -n 1000 localhost get:/:b@10:ir,@1 + +The request specified here is a valid GET with a body consisting of 10 random bytes, +but with 1 random byte inserted in a random place. This could be in the headers, +in the initial request line, or in the body itself. There are a few things +to note here: + +- Corrupting the request in this way will often make the server enter a state where + it's awaiting more input from the client. This is where the + **-t** option comes in, which sets a timeout that causes pathoc to + disconnect after two seconds. +- The **-n** option tells pathoc to repeat the request 1000 times. +- The **-I** option tells pathoc to ignore HTTP 200 response codes. + You can use this to fine-tune what pathoc considers to be an exceptional + condition, and therefore log-worthy. +- The **-e** option tells pathoc to print an explanation of each logged + request, in the form of an expanded pathoc specification with all random + portions and automatic header additions resolved. This lets you precisely + replay a request that triggered an error. + + +Interacting with Proxies +^^^^^^^^^^^^^^^^^^^^^^^^ + +Pathoc has a reasonably sophisticated suite of features for interacting with +proxies. The proxy request syntax very closely mirrors that of straight HTTP, +which means that it is possible to make proxy-style requests using pathoc +without any additional syntax, by simply specifying a full URL instead of a +simple path: + +>>> pathoc -p 8080 localhost "get:'http://google.com'" + +Another common use case is to use an HTTP CONNECT request to probe remote +servers via a proxy. This is done with the **-c** command-line option, which +allows you to specify a remote host and port pair: + +>>> pathoc -c google.com:80 -p 8080 localhost get:/ + +Note that pathoc does **not** negotiate SSL without being explictly instructed +to do so. If you're making a CONNECT request to an SSL-protected resource, you +must also pass the **-s** flag: + +>>> pathoc -sc google.com:443 -p 8080 localhost get:/ + + + +Embedded response specification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One interesting feature of the Request specification language is that you can +embed a response specification in it, which is then added to the request path. +Here's an example: + +>>> pathoc localhost:9999 "get:/p/:s'401:ir,@1'" + +This crafts a request that connects to the pathod server, and which then crafts +a response that generates a 401, with one random byte embedded at a random +point. The response specification is parsed and expanded by pathoc, so you see +syntax errors immediately. This really becomes handy when combined with the +**-e** flag to show the expanded request:: + + 07-06-16 12:32:01: >> 'GET':/p/:s'401:i35,\x27\\x1b\x27:h\x27Content-Length\x27=\x270\x27:h\x27Content-Length\x27=\x270\x27':h'Host'='localhost' + << 401 Unauthorized: 0 bytes + +Note that the embedded response has been resolved *before* being sent to +the server, so that "ir,@1" (embed a random byte at a random location) has +become "i15,\'o\'" (embed the character "o" at offset 15). You now have a +pathoc request specification that is precisely reproducible, even with random +components. This feature comes in terribly handy when testing a proxy, since +you can now drive the server response completely from the client, and have a +complete log of reproducible requests to analyze afterwards. + + +Request Examples +---------------- + +.. list-table:: + :widths: 50 50 + :header-rows: 0 + + * - get:/ + - Get path / + + * - get:/:b@100 + - 100 random bytes as the body + + * - get:/:h"Etag"="&;drop table browsers;" + - Add a header + + * - get:/:u"&;drop table browsers;" + - Add a User-Agent header + + * - get:/:b@100:dr + - Drop the connection randomly + + * - get:/:b@100,ascii:ir,@1 + - 100 ASCII bytes as the body, and randomly inject a random byte + + * - ws:/ + - Initiate a websocket handshake. + + +Response Examples +----------------- + +.. list-table:: + :widths: 50 50 + :header-rows: 0 + + + * - 200 + - A basic HTTP 200 response. + + * - 200:r + - A basic HTTP 200 response with no Content-Length header. This will hang. + + * - 200:da + - Server-side disconnect after all content has been sent. + + * - 200:b\@100 + - 100 random bytes as the body. A Content-Length header is added, so the disconnect + is no longer needed. + + * - 200:b\@100:h"Etag"="';drop table servers;" + - Add a Server header + + * - 200:b\@100:dr + - Drop the connection randomly + + * - 200:b\@100,ascii:ir,@1 + - 100 ASCII bytes as the body, and randomly inject a random byte + + * - 200:b\@1k:c"text/json" + - 1k of random bytes, with a text/json content type + + * - 200:b\@1k:p50,120 + - 1k of random bytes, pause for 120 seconds after 50 bytes + + * - 200:b\@1k:pr,f + - 1k of random bytes, but hang forever at a random location + + * - 200:b\@100:h\@1k,ascii_letters='foo' + - 100 ASCII bytes as the body, randomly generated 100k header name, with the value + 'foo'. diff --git a/docs/pathod/language.rst b/docs/pathod/language.rst new file mode 100644 index 00000000..672e21b6 --- /dev/null +++ b/docs/pathod/language.rst @@ -0,0 +1,257 @@ +.. _language: + +language spec +============= + +************ +HTTP Request +************ + + **method:path:[colon-separated list of features]** + +.. list-table:: + :widths: 20 80 + :header-rows: 0 + + * - method + - A :ref:`VALUE` specifying the HTTP method to + use. Standard methods do not need to be enclosed in quotes, while + non-standard methods can be specified as quoted strings. + + The special method **ws** creates a valid websocket upgrade + GET request, and signals to pathoc to switch to websocket recieve + mode if the server responds correctly. Apart from that, websocket + requests are just like any other, and all aspects of the request + can be over-ridden. + * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\ + - Set a header. + * - r + - Set the **raw** flag on this response. Pathod will not calculate a + *Content-Length* header if a body is set. + * - c\ :ref:`VALUE` + - A shortcut for setting the Content-Type header. Equivalent to + ``h"Content-Type"=VALUE`` + * - u\ :ref:`VALUE` + uSHORTCUT + - Set a User-Agent header on this request. You can specify either a + complete :ref:`VALUE`, or a User-Agent shortcut: **android**, + **blackberry**, **bingbot**, **chrome**, **firefox**, **googlebot**, + **ie9**, **ipad**, **iphone**, **safari**. + * - b\ :ref:`VALUE` + - Set the body. The appropriate Content-Length header is added + automatically unless the **r** flag is set. + * - s\ :ref:`VALUE` + - An embedded Response specification, appended to the path of the request. + * - x\ :ref:`INTEGER` + - Repeat this message N times. + * - d\ :ref:`OFFSET` + - Disconnect after OFFSET bytes (HTTP/1 only). + * - i\ :ref:`OFFSET`,\ :ref:`VALUE` + - Inject the specified value at the offset (HTTP/1 only) + * - p\ :ref:`OFFSET`,SECONDS + - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer + or "f" to pause forever (HTTP/1 only) + + +************* +HTTP Response +************* + + **code:[colon-separated list of features]** + +.. list-table:: + :widths: 20 80 + :header-rows: 0 + + * - code + - An integer specifying the HTTP response code. + + The special method **ws** creates a valid websocket upgrade + response (code 101), and moves pathod to websocket mode. Apart + from that, websocket responses are just like any other, and all + aspects of the response can be over-ridden. + * - m\ :ref:`VALUE` + - HTTP Reason message. Automatically chosen according to the response + code if not specified. (HTTP/1 only) + * - h\:\ :ref:`VALUE`\ =\ :ref:`VALUE`\ + - Set a header. + * - r + - Set the **raw** flag on this response. Pathod will not calculate a + *Content-Length* header if a body is set. + * - l\ :ref:`VALUE` + - A shortcut for setting the Location header. Equivalent to + ``h"Location"=VALUE`` + * - c\ :ref:`VALUE` + - A shortcut for setting the Content-Type header. Equivalent to + ``h"Content-Type"=VALUE`` + * - b\ :ref:`VALUE` + - Set the body. The appropriate Content-Length header is added + automatically unless the **r** flag is set. + * - d\ :ref:`OFFSET` + - Disconnect after OFFSET bytes (HTTP/1 only). + * - i\ :ref:`OFFSET`,\ :ref:`VALUE` + - Inject the specified value at the offset (HTTP/1 only) + * - p\ :ref:`OFFSET`,SECONDS + - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer + or "f" to pause forever (HTTP/1 only) + +*************** +Websocket Frame +*************** + + **wf:[colon-separated list of features]** + +.. list-table:: + :widths: 20 80 + :header-rows: 0 + + * - b\ :ref:`VALUE` + - Set the frame payload. If a masking key is present, the value is + encoded automatically. + * - c\ :ref:`INTEGER` + - Set the op code. This can either be an integer from 0-15, or be one of + the following opcode names: **text** (the default), **continue**, + **binary**, **close**, **ping**, **pong**. + * - d\ :ref:`OFFSET` + - Disconnect after OFFSET bytes + * - i\ :ref:`OFFSET`,\ :ref:`VALUE` + - Inject the specified value at the offset + * - p\ :ref:`OFFSET`,SECONDS + - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer + or "f" to pause forever + * - x\ :ref:`INTEGER` + - Repeat this message N times. + * - [-]fin + - Set or un-set the **fin** bit. + * - k\ :ref:`VALUE` + - Set the masking key. The resulting value must be exactly 4 bytes long. + The special form **knone** specifies that no key should be set, even if + the mask bit is on. + * - l\ :ref:`INTEGER` + - Set the payload length in the frame header, regardless of the actual + body length. + * - [-]mask + - Set or un-set the <b>mask</b> bit. + * - r\ :ref:`VALUE` + - Set the raw frame payload. This disables masking, even if the key is present. + * - [-]rsv1 + - Set or un-set the **rsv1** bit. + * - [-]rsv2 + - Set or un-set the **rsv2** bit. + * - [-]rsv2 + - Set or un-set the **rsv2** bit. + + + +********** +Data types +********** + +.. _INTEGER: + +INTEGER +^^^^^^^ + +.. _OFFSET: + +OFFSET +^^^^^^ + +Offsets are calculated relative to the base message, before any injections or +other transforms are applied. They have 3 flavors: + +======= ========================== +integer An integer byte offset +**r** A random location +**a** The end of the message +======= ========================== + + +.. _VALUE: + +VALUE +^^^^^ + +Literals +"""""""" + +Literal values are specified as a quoted strings:: + + "foo" + +Either single or double quotes are accepted, and quotes can be escaped with +backslashes within the string:: + + 'fo\'o' + +Literal values can contain Python-style backslash escape sequences:: + + 'foo\r\nbar' + + + +Generated +""""""""" + +An @-symbol lead-in specifies that generated data should be used. There are two +components to a generator specification - a size, and a data type. By default +pathod assumes a data type of "bytes". + +Here's a value specifier for generating 100 bytes:: + + @100 + +You can use standard suffixes to indicate larger values. Here, for instance, is +a specifier for generating 100 megabytes: + + @100m + +Data is generated and served efficiently - if you really want to send a +terabyte of data to a client, pathod can do it. The supported suffixes are: + +========== ==================== +b 1024**0 (bytes) +k 1024**1 (kilobytes) +m 1024**2 (megabytes) +g 1024**3 (gigabytes) +t 1024**4 (terabytes) +========== ==================== + +Data types are separated from the size specification by a comma. This specification +generates 100mb of ASCII:: + + @100m,ascii + +Supported data types are: + +================= ============================================== +ascii All ASCII characters +ascii_letters A-Za-z +ascii_lowercase a-z +ascii_uppercase A-Z +bytes All 256 byte values +digits 0-9 +hexdigits 0-f +octdigits 0-7 +punctuation !"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ and space +whitespace \\t \\n \\x0b \\x0c \\r and space +================= ============================================== + + + +Files +""""" + +You can load a value from a specified file path. To do so, you have to specify a +_staticdir_ option to pathod on the command-line, like so: + +>>> pathod -d ~/myassets + +All paths are relative paths under this directory. File loads are indicated by +starting the value specifier with the left angle bracket:: + + <my/path + +The path value can also be a quoted string, with the same syntax as literals:: + + <"my/path" diff --git a/docs/pathod/library.rst b/docs/pathod/library.rst new file mode 100644 index 00000000..b055d089 --- /dev/null +++ b/docs/pathod/library.rst @@ -0,0 +1,14 @@ +.. _library: + +pathod library +============== + +Behind the pathod and pathoc command-line tools lurks the **pathod** library, a +powerful way to manipulate and serve HTTP requests and responses from code. The +canonical documentation for the library is in the code, and can be accessed +using pydoc. + + +.. literalinclude:: ../../examples/pathod/libpathod_pathoc.py + :caption: examples/pathod/libpathod_pathoc.py + :language: python diff --git a/docs/pathod/test.rst b/docs/pathod/test.rst new file mode 100644 index 00000000..cd6e8a29 --- /dev/null +++ b/docs/pathod/test.rst @@ -0,0 +1,35 @@ +.. _test: + +pathod.test +=========== + +The **pathod.test** module is a light, flexible testing layer for HTTP clients. +It works by firing up a Pathod instance in a separate thread, letting you use +Pathod's full abilities to generate responses, and then query Pathod's internal +logs to establish what happened. All the mechanics of startup, shutdown, finding +free ports and so forth are taken care of for you. + +The canonical docs can be accessed using pydoc: + +>>> pydoc pathod.test + +The remainder of this page demonstrates some common interaction patterns using +<a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are +also applicable with only minor modification to most commonly used Python testing +engines. + + +Context Manager +--------------- + +.. literalinclude:: ../../examples/pathod/test_context.py + :caption: examples/pathod/test_context.py + :language: python + + +One instance per test +--------------------- + +.. literalinclude:: ../../examples/pathod/test_setup.py + :caption: examples/pathod/test_setup.py + :language: python diff --git a/docs/transparent/linux.rst b/docs/transparent/linux.rst index ce79128c..1878008c 100644 --- a/docs/transparent/linux.rst +++ b/docs/transparent/linux.rst @@ -35,7 +35,7 @@ achieve transparent mode. >>> mitmproxy -T --host - The :option:`-T` flag turns on transparent mode, and the :option:`--host` + The ``-T`` flag turns on transparent mode, and the ``--host`` argument tells mitmproxy to use the value of the Host header for URL display. 6. Finally, configure your test device to use the host on which mitmproxy is diff --git a/docs/transparent/osx.rst b/docs/transparent/osx.rst index 1791105f..46f0e2df 100644 --- a/docs/transparent/osx.rst +++ b/docs/transparent/osx.rst @@ -50,7 +50,7 @@ Note that this means we don't support transparent mode for earlier versions of O >>> mitmproxy -T --host - The :option:`-T` flag turns on transparent mode, and the :option:`--host` + The ``-T`` flag turns on transparent mode, and the ``--host`` argument tells mitmproxy to use the value of the Host header for URL display. 8. Finally, configure your test device to use the host on which mitmproxy is diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index acb7fc35..b450c19d 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -136,7 +136,7 @@ def raw_format_flow(f, focus, extended): if extended: req.append( fcol( - utils.format_timestamp(f["req_timestamp"]), + human.format_timestamp(f["req_timestamp"]), "highlight" ) ) diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/console/flowdetailview.py index e2c28e71..2a493b90 100644 --- a/mitmproxy/console/flowdetailview.py +++ b/mitmproxy/console/flowdetailview.py @@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division import urwid -from mitmproxy import utils from mitmproxy.console import common, searchable +from netlib import human def maybe_timestamp(base, attr): if base is not None and getattr(base, attr): - return utils.format_timestamp_with_milli(getattr(base, attr)) + return human.format_timestamp_with_milli(getattr(base, attr)) else: return "active" diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 42061a8c..28c57f06 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -28,7 +28,6 @@ from PIL import ExifTags from PIL import Image from six.moves import cStringIO as StringIO -import mitmproxy.utils from mitmproxy import exceptions from mitmproxy.contrib import jsbeautifier from mitmproxy.contrib.wbxml import ASCommandResponse @@ -62,6 +61,14 @@ VIEW_CUTOFF = 512 KEY_MAX = 30 +def pretty_json(s): + try: + p = json.loads(s) + except ValueError: + return None + return json.dumps(p, sort_keys=True, indent=4) + + def format_dict(d): """ Helper function that transforms the given dictionary into a list of @@ -215,9 +222,9 @@ class ViewJSON(View): content_types = ["application/json"] def __call__(self, data, **metadata): - pretty_json = mitmproxy.utils.pretty_json(data) - if pretty_json: - return "JSON", format_text(pretty_json) + pj = pretty_json(data) + if pj: + return "JSON", format_text(pj) class ViewHTML(View): diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index d97b9498..63bd8d3d 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -93,5 +93,5 @@ class FlowReadException(ProxyException): pass -class ControlException(Exception): +class ControlException(ProxyException): pass diff --git a/mitmproxy/models/connections.py b/mitmproxy/models/connections.py index 6347f488..b8e0567a 100644 --- a/mitmproxy/models/connections.py +++ b/mitmproxy/models/connections.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, print_function, division +import time import copy import os @@ -6,7 +7,6 @@ import os import six from mitmproxy import stateobject -from mitmproxy import utils from netlib import certutils from netlib import tcp @@ -39,7 +39,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.clientcert = None self.ssl_established = None - self.timestamp_start = utils.timestamp() + self.timestamp_start = time.time() self.timestamp_end = None self.timestamp_ssl_setup = None self.protocol = None @@ -97,11 +97,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): def convert_to_ssl(self, *args, **kwargs): super(ClientConnection, self).convert_to_ssl(*args, **kwargs) - self.timestamp_ssl_setup = utils.timestamp() + self.timestamp_ssl_setup = time.time() def finish(self): super(ClientConnection, self).finish() - self.timestamp_end = utils.timestamp() + self.timestamp_end = time.time() class ServerConnection(tcp.TCPClient, stateobject.StateObject): @@ -194,9 +194,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): return copy.copy(self) def connect(self): - self.timestamp_start = utils.timestamp() + self.timestamp_start = time.time() tcp.TCPClient.connect(self) - self.timestamp_tcp_setup = utils.timestamp() + self.timestamp_tcp_setup = time.time() def send(self, message): if isinstance(message, list): @@ -218,11 +218,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni - self.timestamp_ssl_setup = utils.timestamp() + self.timestamp_ssl_setup = time.time() def finish(self): tcp.TCPClient.finish(self) - self.timestamp_end = utils.timestamp() + self.timestamp_end = time.time() ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py index 7b9ec030..e2dac221 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/models/flow.py @@ -1,11 +1,11 @@ from __future__ import absolute_import, print_function, division +import time import copy import uuid from mitmproxy import exceptions from mitmproxy import stateobject -from mitmproxy import utils from mitmproxy import version from mitmproxy.models.connections import ClientConnection from mitmproxy.models.connections import ServerConnection @@ -34,7 +34,7 @@ class Error(stateobject.StateObject): @type timestamp: float """ self.msg = msg - self.timestamp = timestamp or utils.timestamp() + self.timestamp = timestamp or time.time() _stateobject_attributes = dict( msg=str, diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index ae03ab7f..187c17f6 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, print_function, division +import time import sys import traceback @@ -9,7 +10,6 @@ import six import netlib.exceptions from mitmproxy import exceptions from mitmproxy import models -from mitmproxy import utils from mitmproxy.protocol import base from netlib import http from netlib import tcp @@ -265,7 +265,7 @@ class HttpLayer(base.Layer): if callable(flow.response.stream): chunks = flow.response.stream(chunks) self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = utils.timestamp() + flow.response.timestamp_end = time.time() def get_response_from_server(self, flow): def get_response(): @@ -310,7 +310,7 @@ class HttpLayer(base.Layer): flow.request, flow.response )) - flow.response.timestamp_end = utils.timestamp() + flow.response.timestamp_end = time.time() # no further manipulation of self.server_conn beyond this point # we can safely set it as the final attribute value here. diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py index 680bc495..15785c72 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils.py @@ -1,38 +1,8 @@ from __future__ import absolute_import, print_function, division -import datetime -import json -import time - import netlib.utils -def timestamp(): - """ - Returns a serializable UTC timestamp. - """ - return time.time() - - -def format_timestamp(s): - s = time.localtime(s) - d = datetime.datetime.fromtimestamp(time.mktime(s)) - return d.strftime("%Y-%m-%d %H:%M:%S") - - -def format_timestamp_with_milli(s): - d = datetime.datetime.fromtimestamp(s) - return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - - -def pretty_json(s): - try: - p = json.loads(s) - except ValueError: - return None - return json.dumps(p, sort_keys=True, indent=4) - - pkg_data = netlib.utils.Data(__name__) diff --git a/netlib/http/http1/assemble.py b/netlib/http/http1/assemble.py index 00d1563b..511328f1 100644 --- a/netlib/http/http1/assemble.py +++ b/netlib/http/http1/assemble.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, print_function, division -from netlib import utils +import netlib.http.url from netlib import exceptions @@ -82,7 +82,7 @@ def _assemble_request_headers(request_data): """ headers = request_data.headers.copy() if "host" not in headers and request_data.scheme and request_data.host and request_data.port: - headers["host"] = utils.hostport( + headers["host"] = netlib.http.url.hostport( request_data.scheme, request_data.host, request_data.port diff --git a/netlib/http/request.py b/netlib/http/request.py index 91d5f020..01801d42 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -227,7 +227,7 @@ class Request(message.Message): def query(self): # type: () -> multidict.MultiDictView """ - The request query string as an :py:class:`MultiDictView` object. + The request query string as an :py:class:`~netlib.multidict.MultiDictView` object. """ return multidict.MultiDictView( self._get_query, @@ -254,7 +254,7 @@ class Request(message.Message): """ The request cookies. - An empty :py:class:`multidict.MultiDictView` object if the cookie monster ate them all. + An empty :py:class:`~netlib.multidict.MultiDictView` object if the cookie monster ate them all. """ return multidict.MultiDictView( self._get_cookies, @@ -329,7 +329,7 @@ class Request(message.Message): @property def urlencoded_form(self): """ - The URL-encoded form data as an :py:class:`multidict.MultiDictView` object. + The URL-encoded form data as an :py:class:`~netlib.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. """ @@ -359,7 +359,7 @@ class Request(message.Message): @property def multipart_form(self): """ - The multipart form data as an :py:class:`MultipartFormDict` object. + The multipart form data as an :py:class:`~netlib.multidict.MultiDictView` object. None if the content-type indicates non-form data. """ return multidict.MultiDictView( diff --git a/netlib/http/response.py b/netlib/http/response.py index 44b58be6..7dabfcab 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -73,7 +73,7 @@ class Response(message.Message): def cookies(self): # type: () -> multidict.MultiDictView """ - The response cookies. A possibly empty :py:class:`multidict.MultiDictView`, where the keys are + The response cookies. A possibly empty :py:class:`~netlib.multidict.MultiDictView`, where the keys are cookie name strings, and values are (value, attr) tuples. Value is a string, and attr is an ODictCaseless containing cookie attributes. Within attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value. diff --git a/netlib/http/url.py b/netlib/http/url.py index 5d461387..2fc6e7ee 100644 --- a/netlib/http/url.py +++ b/netlib/http/url.py @@ -78,7 +78,7 @@ def unparse(scheme, host, port, path=""): """ if path == "*": path = "" - return "%s://%s%s" % (scheme, utils.hostport(scheme, host, port), path) + return "%s://%s%s" % (scheme, hostport(scheme, host, port), path) def encode(s): @@ -94,3 +94,16 @@ def decode(s): Takes a urlencoded string and returns a list of (key, value) tuples. """ return urllib.parse.parse_qsl(s, keep_blank_values=True) + + +def hostport(scheme, host, port): + """ + Returns the host component, with a port specifcation if needed. + """ + if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]: + return host + else: + if isinstance(host, six.binary_type): + return b"%s:%d" % (host, port) + else: + return "%s:%d" % (host, port) diff --git a/netlib/human.py b/netlib/human.py index a007adc7..72e96d30 100644 --- a/netlib/human.py +++ b/netlib/human.py @@ -1,3 +1,6 @@ +import datetime +import time + SIZE_TABLE = [ ("b", 1024 ** 0), @@ -48,3 +51,14 @@ def pretty_duration(secs): return formatter.format(secs) # less than 1 sec return "{:.0f}ms".format(secs * 1000) + + +def format_timestamp(s): + s = time.localtime(s) + d = datetime.datetime.fromtimestamp(time.mktime(s)) + return d.strftime("%Y-%m-%d %H:%M:%S") + + +def format_timestamp_with_milli(s): + d = datetime.datetime.fromtimestamp(s) + return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] diff --git a/netlib/tcp.py b/netlib/tcp.py index de12102e..0eec326b 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -580,8 +580,10 @@ class _Connection(object): @contextlib.contextmanager def _closer(client): - yield - client.close() + try: + yield + finally: + client.close() class TCPClient(_Connection): diff --git a/netlib/utils.py b/netlib/utils.py index b4b99679..79340cbd 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -4,8 +4,6 @@ import re import importlib import inspect -import six - def setbit(byte, offset, value): """ @@ -94,16 +92,3 @@ def is_valid_host(host): def is_valid_port(port): return 0 <= port <= 65535 - - -def hostport(scheme, host, port): - """ - Returns the host component, with a port specifcation if needed. - """ - if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]: - return host - else: - if isinstance(host, six.binary_type): - return b"%s:%d" % (host, port) - else: - return "%s:%d" % (host, port) diff --git a/pathod/language/base.py b/pathod/language/base.py index 1369a3c7..25f3fd1a 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -261,7 +261,7 @@ class _Component(Token): """ A value component of the primary specification of an message. - Components produce byte values desribe the bytes of the message. + Components produce byte values describing the bytes of the message. """ def values(self, settings): # pragma: no cover @@ -272,9 +272,9 @@ class _Component(Token): def string(self, settings=None): """ - A string representation of the object. + A bytestring representation of the object. """ - return "".join(i[:] for i in self.values(settings or {})) + return b"".join(i[:] for i in self.values(settings or {})) class KeyValue(_Component): @@ -391,7 +391,7 @@ class Integer(_Component): "Integer value must be between %s and %s." % self.bounds, 0, 0 ) - self.value = str(value) + self.value = str(value).encode() @classmethod def expr(cls): @@ -401,10 +401,10 @@ class Integer(_Component): return e.setParseAction(lambda x: cls(*x)) def values(self, settings): - return self.value + return [self.value] def spec(self): - return "%s%s" % (self.preamble, self.value) + return "%s%s" % (self.preamble, self.value.decode()) def freeze(self, settings_): return self @@ -555,7 +555,7 @@ class NestedMessage(Token): try: self.parsed = self.nest_type( self.nest_type.expr().parseString( - value.val, + value.val.decode(), parseAll=True ) ) @@ -578,4 +578,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f))) + return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode()))) diff --git a/pathod/language/http.py b/pathod/language/http.py index b2308d5e..4cc7db5f 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -57,7 +57,7 @@ class _HeaderMixin(object): unique_name = None def format_header(self, key, value): - return [key, ": ", value, "\r\n"] + return [key, b": ", value, b"\r\n"] def values(self, settings): return self.format_header( @@ -88,7 +88,7 @@ class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): def values(self, settings): value = self.value.val if self.option_used: - value = user_agents.get_by_shortcut(value.lower())[2] + value = user_agents.get_by_shortcut(value.lower().decode())[2].encode() return self.format_header( self.key.get_generator(settings), @@ -109,7 +109,7 @@ def get_header(val, headers): class _HTTPMessage(message.Message): - version = "HTTP/1.1" + version = b"HTTP/1.1" @property def actions(self): @@ -133,10 +133,10 @@ class _HTTPMessage(message.Message): def values(self, settings): vals = self.preamble(settings) - vals.append("\r\n") + vals.append(b"\r\n") for h in self.headers: vals.extend(h.values(settings)) - vals.append("\r\n") + vals.append(b"\r\n") if self.body: vals.extend(self.body.values(settings)) return vals @@ -171,18 +171,18 @@ class Response(_HTTPMessage): return self.tok(Reason) def preamble(self, settings): - l = [self.version, " "] + l = [self.version, b" "] l.extend(self.status_code.values(settings)) status_code = int(self.status_code.value) - l.append(" ") + l.append(b" ") if self.reason: l.extend(self.reason.values(settings)) else: l.append( status_codes.RESPONSES.get( status_code, - "Unknown code" - ) + b"Unknown code" + ).encode() ) return l @@ -205,8 +205,8 @@ class Response(_HTTPMessage): if not get_header(i[0], self.headers): tokens.append( Header( - base.TokValueLiteral(i[0]), - base.TokValueLiteral(i[1])) + base.TokValueLiteral(i[0].decode()), + base.TokValueLiteral(i[1].decode())) ) if not self.raw: if not get_header("Content-Length", self.headers): @@ -294,11 +294,11 @@ class Request(_HTTPMessage): def preamble(self, settings): v = self.method.values(settings) - v.append(" ") + v.append(b" ") v.extend(self.path.values(settings)) if self.nested_response: v.append(self.nested_response.parsed.spec()) - v.append(" ") + v.append(b" ") v.append(self.version) return v @@ -314,8 +314,8 @@ class Request(_HTTPMessage): if not get_header(i[0], self.headers): tokens.append( Header( - base.TokValueLiteral(i[0]), - base.TokValueLiteral(i[1]) + base.TokValueLiteral(i[0].decode()), + base.TokValueLiteral(i[1].decode()) ) ) if not self.raw: diff --git a/pathod/language/message.py b/pathod/language/message.py index 33124856..fea4f4de 100644 --- a/pathod/language/message.py +++ b/pathod/language/message.py @@ -1,5 +1,6 @@ import abc from . import actions, exceptions +from netlib import strutils LOG_TRUNCATE = 1024 @@ -49,7 +50,7 @@ class Message(object): def preview_safe(self): """ - Return a copy of this message that issafe for previews. + Return a copy of this message that is safe for previews. """ tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)] return self.__class__(tokens) @@ -80,10 +81,10 @@ class Message(object): # We truncate at 1k. if hasattr(v, "values"): v = [x[:LOG_TRUNCATE] for x in v.values(settings)] - v = "".join(v).encode("string_escape") + v = strutils.bytes_to_escaped_str(b"".join(v)) elif hasattr(v, "__len__"): v = v[:LOG_TRUNCATE] - v = v.encode("string_escape") + v = strutils.bytes_to_escaped_str(v) ret[i] = v ret["spec"] = self.spec() return ret diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py index 5ca2ca85..5bb0cdd3 100644 --- a/pathod/pathod_cmdline.py +++ b/pathod/pathod_cmdline.py @@ -5,7 +5,16 @@ import os.path import re from netlib import tcp, human -from . import pathod, version, utils +from . import pathod, version + + +def parse_anchor_spec(s): + """ + Return a tuple, or None on error. + """ + if "=" not in s: + return None + return tuple(s.split("=", 1)) def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): @@ -188,7 +197,7 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): alst = [] for i in args.anchors: - parts = utils.parse_anchor_spec(i) + parts = parse_anchor_spec(i) if not parts: return parser.error("Invalid anchor specification: %s" % i) alst.append(parts) diff --git a/pathod/templates/about.html b/pathod/templates/about.html deleted file mode 100644 index 340dc386..00000000 --- a/pathod/templates/about.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<section> - <div class="page-header"> - <h1>About</h1> - </div> - <div class="row"> - <div class="span6"> - <div> - <p>pathod is developed by <a href="http://corte.si">Aldo Cortesi</a>.</p> - </div> - - <div> - <ul> - <li>email: <a href="mailto:aldo@corte.si">aldo@corte.si</a></li> - <li>twitter: <a href="http://twitter.com/cortesi">@cortesi</a></li> - <li>github: <a href="https://github.com/cortesi">github.com/cortesi</a></li> - </ul> - </div> - </div> - </div> -</section> -{% endblock %} diff --git a/pathod/templates/docframe.html b/pathod/templates/docframe.html deleted file mode 100644 index 797de20a..00000000 --- a/pathod/templates/docframe.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "layout.html" %} - -{% macro subs(s) %} - {% if subsection == s %} - class="active" - {% endif %} -{% endmacro %} - -{% block content %} -<div class="row"> - <div class="span3"> - <div class="well sidebar-nav"> - <ul class="nav nav-list"> - <li {{subs( "pathod")}}><a href="/docs/pathod">pathod</a></li> - <li {{subs( "pathoc")}}><a href="/docs/pathoc">pathoc</a></li> - <li {{subs( "lang")}}><a href="/docs/language">language</a></li> - <li {{subs( "pathod")}}><a href="/docs/pathod">pathod</a></li> - <li {{subs( "test")}}><a href="/docs/test">pathod.test</a></li> - </ul> - </div> - </div> - <div class="span9"> - {% block body %} {% endblock %} - </div> -</div> -{% endblock %} diff --git a/pathod/templates/docs_lang.html b/pathod/templates/docs_lang.html deleted file mode 100644 index a1d22aef..00000000 --- a/pathod/templates/docs_lang.html +++ /dev/null @@ -1,196 +0,0 @@ -{% extends "docframe.html" %} {% block body %} -<div class="page-header"> - <h1> - Language Spec - <small>The mini-language at the heart of pathoc and pathod.</small> - </h1> -</div> - -<ul class="nav nav-tabs"> - <li class="active"><a href="#specifying_requests" data-toggle="tab">HTTP Requests</a></li> - <li><a href="#specifying_responses" data-toggle="tab">HTTP Responses</a></li> - <li><a href="#websockets" data-toggle="tab">Websocket Frames</a></li> -</ul> - -<div class="tab-content"> - <div class="tab-pane" id="specifying_responses"> - {% include "docs_lang_responses.html" %} - </div> - <div class="tab-pane active" id="specifying_requests"> - {% include "docs_lang_requests.html" %} - </div> - <div class="tab-pane" id="websockets"> - {% include "docs_lang_websockets.html" %} - </div> -</div> - -<section id="features"> - <div class="page-header"> - <h1>Features</h1> - </div> - - <a id="offsetspec"></a> - <h2>OFFSET</h2> - - <p> - Offsets are calculated relative to the base message, before any injections or other - transforms are applied. They have 3 flavors: - </p> - - <ul> - <li>An integer byte offset </li> - <li><b>r</b> for a random location</li> - <li><b>a</b> for the end of the message</li> - </ul> - - <a id="valuespec"></a> - <h2>VALUE</h2> - - <h3>Literals</h3> - - <p>Literal values are specified as a quoted strings: </p> - - <pre class="example">"foo"</pre> - - <p> - Either single or double quotes are accepted, and quotes can be escaped with backslashes - within the string: - </p> - - <pre class="example">'fo\'o'</pre> - - <p>Literal values can contain Python-style backslash escape sequences:</p> - - <pre class="example">'foo\r\nbar'</pre> - - <h3>Files</h3> - - <p> - You can load a value from a specified file path. To do so, you have to specify a - _staticdir_ option to pathod on the command-line, like so: - </p> - - <pre class="example">pathod -d ~/myassets</pre> - - <p> - All paths are relative paths under this directory. File loads are indicated by starting - the value specifier with the left angle bracket: - </p> - - <pre class="example"><my/path</pre> - - <p>The path value can also be a quoted string, with the same syntax as literals:</p> - - <pre class="example"><"my/path"</pre> - - - <h3>Generated values</h3> - - <p> - An @-symbol lead-in specifies that generated data should be used. There are two components - to a generator specification - a size, and a data type. By default pathod - assumes a data type of "bytes". - </p> - - <p>Here's a value specifier for generating 100 bytes: - - <pre class="example">@100</pre> - </p> - - <p> - You can use standard suffixes to indicate larger values. Here, for instance, is a - specifier for generating 100 megabytes: - </p> - - <pre class="example">@100m</pre> - - <p> - Data is generated and served efficiently - if you really want to send a terabyte - of data to a client, pathod can do it. The supported suffixes are: - </p> - - <table class="table table-bordered"> - <tbody> - <tr> - <td>b</td> - <td>1024**0 (bytes)</td> - </tr> - <tr> - <td>k</td> - <td>1024**1 (kilobytes)</td> - </tr> - <tr> - <td>m</td> - <td>1024**2 (megabytes)</td> - </tr> - <tr> - <td>g</td> - <td>1024**3 (gigabytes)</td> - </tr> - <tr> - <td>t</td> - <td>1024**4 (terabytes)</td> - </tr> - </tbody> - </table> - - <p> - Data types are separated from the size specification by a comma. This specification - generates 100mb of ASCII: - </p> - - <pre class="example">@100m,ascii</pre> - - <p>Supported data types are:</p> - - <table class="table table-bordered"> - <tbody> - <tr> - <td>ascii</td> - <td>All ASCII characters</td> - </tr> - <tr> - <td>ascii_letters</td> - <td>A-Za-z</td> - </tr> - <tr> - <td>ascii_lowercase</td> - <td>a-z</td> - </tr> - <tr> - <td>ascii_uppercase</td> - <td>A-Z</td> - </tr> - <tr> - <td>bytes</td> - <td>All 256 byte values</td> - </tr> - <tr> - <td>digits</td> - <td>0-9</td> - </tr> - <tr> - <td>hexdigits</td> - <td>0-f</td> - </tr> - <tr> - <td>octdigits</td> - <td>0-7</td> - </tr> - <tr> - <td>punctuation</td> - <td> - <pre>!"#$%&\'()*+,-./:; - <=>?@[\\]^_`{|}~</pre> - </td> - </tr> - <tr> - <td>whitespace</td> - <td> - <pre>\t\n\x0b\x0c\r and space</pre> - </td> - </tr> - </tbody> - </table> -</section> -{% endblock %} diff --git a/pathod/templates/docs_lang_requests.html b/pathod/templates/docs_lang_requests.html deleted file mode 100644 index 81aff535..00000000 --- a/pathod/templates/docs_lang_requests.html +++ /dev/null @@ -1,114 +0,0 @@ -<pre class="example">method:path:[colon-separated list of features]</pre> -</p> - -<table class="table table-bordered"> - <tbody> - <tr> - <td>method</td> - <td> - <p> - A <a href="#valuespec">VALUE</a> specifying the HTTP method to - use. Standard methods do not need to be enclosed in quotes, while - non-standard methods can be specified as quoted strings. - </p> - - <p> - The special method <b>ws</b> creates a valid websocket upgrade - GET request, and signals to pathoc to switch to websocket recieve - mode if the server responds correctly. Apart from that, websocket - requests are just like any other, and all aspects of the request - can be over-ridden. - </p> - </td> - </tr> - - <tr> - <td>h<a href="#valuespec">VALUE</a>=<a href="#valuespec">VALUE</a></td> - <td> - Set a header. - </td> - </tr> - - <tr> - <td>r</td> - <td> - Set the "raw" flag on this response. Pathod will not calculate a Content-Length header - if a body is set. - </td> - </tr> - - <tr> - <td>c<a href="#valuespec">VALUE</a></td> - <td> - A shortcut for setting the Content-Type header. Equivalent to h"Content-Type"=VALUE - </td> - </tr> - - <tr> - <td>u<a href="#valuespec">VALUE</a> - <br> uSHORTCUT - </td> - - <td> - Set a User-Agent header on this request. You can specify either a complete - <a href="#valuespec">VALUE</a>, or a User-Agent shortcut: - - <table class="table table-condensed"> - {% for i in uastrings %} - <tr> - <td><b>{{ i[1] }}</b></td> - <td>{{ i[0] }}</td> - </tr> - {% endfor %} - </table> - </td> - </tr> - - <tr> - <td>b<a href="#valuespec">VALUE</a></td> - <td> - Set the body. The appropriate Content-Length header is added automatically unless - the "r" flag is set. - </td> - </tr> - - <tr> - <td>s<a href="#valuespec">VALUE</a></td> - <td> - An embedded Response specification, appended to the path of the request. - </td> - </tr> - - <tr> - <td>x<a href="#valuespec">INTEGER</a></td> - <td> - Repeat this message N times. - </td> - </tr> - - <tr> - <td>d<a href="#offsetspec">OFFSET</a></td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Disconnect after - OFFSET bytes. - </td> - </tr> - - <tr> - <td>i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a></td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Inject the specified - value at the offset. - </td> - </tr> - - <tr> - <td>p<a href="#offsetspec">OFFSET</a>,SECONDS</td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Pause for SECONDS - seconds after OFFSET bytes. SECONDS can be an integer or "f" to pause - forever. - </td> - </tr> - </tbody> -</table> diff --git a/pathod/templates/docs_lang_responses.html b/pathod/templates/docs_lang_responses.html deleted file mode 100644 index 9a85ff1a..00000000 --- a/pathod/templates/docs_lang_responses.html +++ /dev/null @@ -1,88 +0,0 @@ -<pre class="example">code:[colon-separated list of features]</pre> - -<table class="table table-bordered"> - <tbody> - <tr> - <td>code</td> - <td> - <p>An integer specifying the HTTP response code.</p> - <p> - The special method <b>ws</b> creates a valid websocket upgrade - response (code 101), and moves pathod to websocket mode. Apart - from that, websocket responses are just like any other, and all - aspects of the response can be over-ridden. - </p> - </td> - </tr> - - <tr> - <td>m<a href="#valuespec">VALUE</a></td> - <td> - <span class="badge badge-info">HTTP/1 only</span> HTTP Reason message. - Automatically chosen according to the response code if not specified. - </td> - </tr> - - <tr> - <td>h<a href="#valuespec">VALUE</a>=<a href="#valuespec">VALUE</a></td> - <td> - Set a header. - </td> - </tr> - - <tr> - <td>r</td> - <td> - Set the "raw" flag on this response. Pathod will not calculate a Content-Length header - if a body is set, or add a Date header to the response. - </td> - </tr> - - <tr> - <td>l<a href="#valuespec">VALUE</a></td> - <td> - A shortcut for setting the Location header. Equivalent to h"Location"=VALUE - </td> - </tr> - - <tr> - <td>c<a href="#valuespec">VALUE</a></td> - <td> - A shortcut for setting the Content-Type header. Equivalent to h"Content-Type"=VALUE - </td> - </tr> - - <tr> - <td>b<a href="#valuespec">VALUE</a></td> - <td> - Set the body. The appropriate Content-Length header is added automatically unless - the "r" flag is set. - </td> - </tr> - - <tr> - <td>d<a href="#offsetspec">OFFSET</a></td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Disconnect after - OFFSET bytes. - </td> - </tr> - - <tr> - <td>i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a></td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Inject the specified - value at the offset. - </td> - </tr> - - <tr> - <td>p<a href="#offsetspec">OFFSET</a>,SECONDS</td> - <td> - <span class="badge badge-info">HTTP/1 only</span> Pause for SECONDS - seconds after OFFSET bytes. SECONDS can be an integer or "f" to pause - forever. - </td> - </tr> - </tbody> -</table> diff --git a/pathod/templates/docs_lang_websockets.html b/pathod/templates/docs_lang_websockets.html deleted file mode 100644 index dd318e0b..00000000 --- a/pathod/templates/docs_lang_websockets.html +++ /dev/null @@ -1,115 +0,0 @@ -<pre class="example">wf:[colon-separated list of features]</pre> -</p> - -<table class="table table-bordered"> - <tbody> - - <tr> - <td> b<a href="#valuespec">VALUE</a> </td> - <td> - Set the frame payload. If a masking key is present, the value is encoded automatically. - </td> - </tr> - - <tr> - <td> c<a href="#valuespec">INTEGER</a> </td> - <td> - - Set the op code. This can either be an integer from 0-15, or be one of the following - opcode names: <b>text</b> (the default), - <b>continue</b>, <b>binary</b>, <b>close</b>, <b>ping</b>, - <b>pong</b>. - - </td> - </tr> - - <tr> - <td> d<a href="#offsetspec">OFFSET</a> </td> - <td> - Disconnect after OFFSET bytes. - </td> - </tr> - - <tr> - <td> [-]fin </td> - <td> - Set or un-set the <b>fin</b> bit. - </td> - </tr> - - <tr> - <td> i<a href="#offsetspec">OFFSET</a>,<a href="#valuespec">VALUE</a> </td> - <td> - Inject the specified value at the offset. - </td> - </tr> - - <tr> - <td> k<a href="#valuespec">VALUE</a> </td> - <td> - Set the masking key. The resulting value must be exactly 4 bytes long. The special - form - <b>knone</b> specifies that no key should be set, even if the mask - bit is on. - </td> - </tr> - - <tr> - <td> l<a href="#valuespec">INTEGER</a> </td> - <td> - Set the payload length in the frame header, regardless of the actual body length. - </td> - </tr> - - <tr> - <td> [-]mask </td> - <td> - Set or un-set the <b>mask</b> bit. - </td> - </tr> - - <tr> - <td> p<a href="#offsetspec">OFFSET</a>,SECONDS </td> - <td> - Pause for SECONDS seconds after OFFSET bytes. SECONDS can be an integer or "f" to - pause forever. - </td> - </tr> - - <tr> - <td> r<a href="#valuespec">VALUE</a> </td> - <td> - Set the raw frame payload. This disables masking, even if the key is present. - </td> - </tr> - - <tr> - <td> [-]rsv1 </td> - <td> - Set or un-set the <b>rsv1</b> bit. - </td> - </tr> - - <tr> - <td> [-]rsv2 </td> - <td> - Set or un-set the <b>rsv2</b> bit. - </td> - </tr> - - <tr> - <td> [-]rsv3 </td> - <td> - Set or un-set the <b>rsv3</b> bit. - </td> - </tr> - - <tr> - <td> x<a href="#valuespec">INTEGER</a> </td> - <td> - Repeat this message N times. - </td> - </tr> - - </tbody> -</table> diff --git a/pathod/templates/docs_libpathod.html b/pathod/templates/docs_libpathod.html deleted file mode 100644 index 8eb6846a..00000000 --- a/pathod/templates/docs_libpathod.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "docframe.html" %} {% block body %} -<div class="page-header"> - <h1> - pathod - <small>Using pathod and pathoc in code.</small> - </h1> -</div> - -<div class="row"> - <div class="span6"> - <p> - Behind the pathod and pathoc command-line tools lurks <b>pathod</b>, - a powerful library for manipulating and serving HTTP requests and responses. - The canonical documentation for the library is in the code, and can be - accessed using pydoc. - </p> - </div> - <div class="span6"> - <h1>pathoc</h1> - {% include "pathod_pathoc.html" %} - </div> -</div> -{% endblock %} diff --git a/pathod/templates/docs_pathoc.html b/pathod/templates/docs_pathoc.html deleted file mode 100644 index d38c3a77..00000000 --- a/pathod/templates/docs_pathoc.html +++ /dev/null @@ -1,211 +0,0 @@ -{% extends "docframe.html" %} {% block body %} -<div class="page-header"> - <h1> - pathoc - <small>A perverse HTTP client.</small> - </h1> -</div> - -<p> - Pathoc is a perverse HTTP daemon designed to let you craft almost any conceivable - HTTP request, including ones that creatively violate the standards. HTTP requests - are specified using a - <a href="/docs/language">small, terse language</a>, which pathod shares with - its server-side twin <a href="/docs/pathod">pathod</a>. To view pathoc's complete - range of options, use the command-line help: -</p> - -<pre class="terminal">pathoc --help</pre> - -<section> - <div class="page-header"> - <h1>Getting Started</h1> - </div> - - <p>The basic pattern for pathoc commands is as follows: </p> - - <pre class="terminal">pathoc hostname request [request ...]</pre> - - <p> - That is, we specify the hostname to connect to, followed by one or more requests. - Lets start with a simple example: - </p> - - <pre class="terminal"> - > pathoc google.com get:/ << 301 Moved Permanently: 219 bytes - </pre> - - <p> - Here, we make a GET request to the path / on port 80 of google.com. Pathoc's output - tells us that the server responded with a 301. We can tell pathoc to connect - using SSL, in which case the default port is changed to 443 (you can over-ride - the default port with the <b>-p</b> command-line option): - </p> - - <pre class="terminal"> - > pathoc -s google.com get:/ << 301 Moved Permanently: 219 bytes - </pre> -</section> - - -<section> - <div class="page-header"> - <h1>Multiple Requests</h1> - </div> - - <p> - There are two ways to tell pathoc to issue multiple requests. The first is to specify - them on the command-line, like so: - </p> - - <pre class="terminal"> - > pathoc google.com get:/ get:/ << 301 Moved Permanently: 219 bytes << - 301 Moved Permanently: 219 bytes - </pre> - - <p> - In this case, pathoc issues the specified requests over the same TCP connection - - so in the above example only one connection is made to google.com - </p> - - <p>The other way to issue multiple requets is to use the <b>-n</b> flag:</p> - - <pre class="terminal"> - > pathoc -n 2 google.com get:/ << 301 Moved Permanently: 219 bytes << 301 - Moved Permanently: 219 bytes - </pre> - - <p> - The output is identical, but two separate TCP connections are made to the upstream - server. These two specification styles can be combined: - </p> - - <pre class="terminal"> - > pathoc -n 2 google.com get:/ get:/ << 301 Moved Permanently: 219 bytes << - 301 Moved Permanently: 219 bytes << 301 Moved Permanently: 219 bytes << - 301 Moved Permanently: 219 bytes - </pre> - - <p>Here, two distinct TCP connections are made, with two requests issued over each.</p> -</section> - - -<section> - <div class="page-header"> - <h1>Basic Fuzzing</h1> - </div> - - <p> - The combination of pathoc's powerful request specification language and a few of - its command-line options makes for quite a powerful basic fuzzer. Here's - an example: - </p> - - <pre class="terminal"> - > pathoc -e -I 200 -t 2 -n 1000 localhost get:/:b@10:ir,@1 - </pre> - - <p> - The request specified here is a valid GET with a body consisting of 10 random bytes, - but with 1 random byte inserted in a random place. This could be in the headers, - in the initial request line, or in the body itself. There are a few things - to note here: - </p> - - <ul> - <li> - Corrupting the request in this way will often make the server enter a state where - it's awaiting more input from the client. This is where the - <b>-t</b> option comes in, which sets a timeout that causes pathoc to - disconnect after two seconds. - </li> - - <li> - The <b>-n</b> option tells pathoc to repeat the request 1000 times. - </li> - - <li> - The <b>-I</b> option tells pathoc to ignore HTTP 200 response codes. - You can use this to fine-tune what pathoc considers to be an exceptional - condition, and therefore log-worthy. - </li> - - <li> - The <b>-e</b> option tells pathoc to print an explanation of each logged - request, in the form of an expanded pathoc specification with all random - portions and automatic header additions resolved. This lets you precisely - replay a request that triggered an error. - </li> - </ul> -</section> - - -<section> - <div class="page-header"> - <h1>Interacting with Proxies</h1> - </div> - - <p> - Pathoc has a reasonably sophisticated suite of features for interacting with proxies. - The proxy request syntax very closely mirrors that of straight HTTP, which - means that it is possible to make proxy-style requests using pathoc without - any additional syntax, by simply specifying a full URL instead of a simple - path: - </p> - - <pre class="terminal">> pathoc -p 8080 localhost "get:'http://google.com'"</pre> - - <p> - Another common use case is to use an HTTP CONNECT request to probe remote servers - via a proxy. This is done with the <b>-c</b> command-line option, - which allows you to specify a remote host and port pair: - </p> - - <pre class="terminal">> pathoc -c google.com:80 -p 8080 localhost get:/</pre> - - <p> - Note that pathoc does <b>not</b> negotiate SSL without being explictly instructed - to do so. If you're making a CONNECT request to an SSL-protected resource, - you must also pass the <b>-s</b> flag: - </p> - - <pre class="terminal">> pathoc -sc google.com:443 -p 8080 localhost get:/</pre> -</section> - - -<section> - <div class="page-header"> - <h1>Embedded response specification</h1> - </div> - - <p> - One interesting feature of the Request sppecification language is that you can embed - a response specifcation in it, which is then added to the request path. Here's - an example: - </p> - - <pre class="terminal">> pathoc localhost:9999 "get:/p/:s'401:ir,@1'"</pre> - - <p> - This crafts a request that connects to the pathod server, and which then crafts a - response that generates a 401, with one random byte embedded at a random - point. The response specification is parsed and expanded by pathoc, so you - see syntax errors immediately. This really becomes handy when combined with - the <b>-e</b> flag to show the expanded request: - </p> - - <pre class="terminal"> - > > pathoc -e localhost:9999 "get:/p/:s'401:ir,@1'" >> Spec: get:/p/:s'401:i15,\'o\':h\'Content-Length\'=\'0\'':h'Content-Length'='0' - << 401 Unoauthorized: 0 bytes </pre> - - <p> - Note that the embedded response has been resolved <i>before</i> being sent - to the server, so that "ir,@1" (embed a random byte at a random location) - has become "i15,\'o\'" (embed the character "o" at offset 15). You now have - a pathoc request specification that is precisely reproducable, even with - random components. This feature comes in terribly handy when testing a proxy, - since you can now drive the server repsonse completely from the client, and - have a complete log of reproducible requests to analyse afterwards. - </p> -</section> -{% endblock %} diff --git a/pathod/templates/docs_pathod.html b/pathod/templates/docs_pathod.html deleted file mode 100644 index 0d0ae933..00000000 --- a/pathod/templates/docs_pathod.html +++ /dev/null @@ -1,172 +0,0 @@ -{% extends "docframe.html" %} {% block body %} -<div class="page-header"> - <h1> - pathod - <small>A pathological web daemon.</small> - </h1> -</div> - -<p> - Pathod is a pathological HTTP daemon designed to let you craft almost any conceivable - HTTP response, including ones that creatively violate the standards. HTTP responses - are specified using a - <a href="/docs/language">small, terse language</a>, which pathod shares with - its evil twin <a href="/docs/pathoc">pathoc</a>. -</p> - -<section> - <div class="page-header"> - <h1>Getting started</h1> - </div> - - <p>To start playing with pathod, simply fire up the daemon:</p> - - <pre class="terminal">./pathod</pre> - - <p> - By default, the service listens on port 9999 of localhost. Pathod's documentation - is self-hosting, and the pathod daemon exposes an interface that lets you - play with the specifciation language, preview what responses and requests - would look like on the wire, and view internal logs. To access all of this, - just fire up your browser, and point it to the following URL: - </p> - - <pre class="example">http://localhost:9999</pre> - - <p> - The default crafting anchor point is the path <b>/p/</b>. Anything after - this URL prefix is treated as a response specifier. So, hitting the following - URL will generate an HTTP 200 response with 100 bytes of random data: - </p> - - <pre class="example">http://localhost:9999/p/200:b@100</pre> - - <p> - See the <a href="/docs/language">language documentation</a> to get (much) - fancier. The pathod daemon also takes a range of configuration options. To - view those, use the command-line help: - </p> - - <pre class="terminal">./pathod --help</pre> - -</section> - -<section> - <div class="page-header"> - <h1>Acting as a proxy</h1> - </div> - - <p> - Pathod automatically responds to both straight HTTP and proxy requests. For proxy - requests, the upstream host is ignored, and the path portion of the URL is - used to match anchors. This lets you test software that supports a proxy - configuration by spoofing responses from upstream servers. - </p> - - <p> - By default, we treat all proxy CONNECT requests as HTTPS traffic, serving the response - using either pathod's built-in certificates, or the cert/key pair specified - by the user. You can over-ride this behaviour if you're testing a client - that makes a non-SSL CONNECT request using the -C command-line option. - </p> -</section> - - -<section> - <div class="page-header"> - <h1>Anchors</h1> - </div> - - <p> - Anchors provide an alternative to specifying the response in the URL. Instead, you - attach a response to a pre-configured anchor point, specified with a regex. - When a URL matching the regex is requested, the specified response is served. - </p> - - <pre class="terminal">./pathod -a "/foo=200"</pre> - - <p> - Here, "/foo" is the regex specifying the anchor path, and the part after the "=" - is a response specifier. - </p> -</section> - - -<section> - <div class="page-header"> - <h1>File Access</h1> - </div> - - <p> - There are two operators in the <a href="/docs/language">language</a> that - load contents from file - the <b>+</b> operator to load an entire request - specification from file, and the <b>></b> value specifier. In pathod, - both of these operators are restricted to a directory specified at startup, - or disabled if no directory is specified:</p> - <pre class="terminal">./pathod -d ~/staticdir"</pre> -</section> - - -<section> - <div class="page-header"> - <h1>Internal Error Responses</h1> - </div> - - <p> - Pathod uses the non-standard 800 response code to indicate internal errors, to distinguish - them from crafted responses. For example, a request to: - </p> - - <pre class="example">http://localhost:9999/p/foo</pre> - - <p> - ... will return an 800 response because "foo" is not a valid page specifier. - </p> -</section> - - -<section> - <div class="page-header"> - <h1>API</h1> - </div> - - <p> - pathod exposes a simple API, intended to make it possible to drive and inspect the - daemon remotely for use in unit testing and the like. - </p> - - <table class="table table-bordered"> - <tbody> - <tr> - <td> - /api/clear_log - </td> - <td> - A POST to this URL clears the log buffer. - </td> - </tr> - <tr> - <td> - /api/info - </td> - <td> - Basic version and configuration info. - </td> - </tr> - <tr> - <td> - /api/log - </td> - <td> - Returns the current log buffer. At the moment the buffer size is 500 entries - when - the log grows larger than this, older entries are discarded. - The returned data is a JSON dictionary, with the form: - - <pre>{ 'log': [ ENTRIES ] } </pre> You can preview the JSON data - returned for a log entry through the built-in web interface. - </td> - </tr> - </tbody> - </table> -</section> -{% endblock %} diff --git a/pathod/templates/docs_test.html b/pathod/templates/docs_test.html deleted file mode 100644 index ecd84bc7..00000000 --- a/pathod/templates/docs_test.html +++ /dev/null @@ -1,50 +0,0 @@ -{% extends "docframe.html" %} {% block body %} -<div class="page-header"> - <h1> - pathod.test - <small>Using pathod in unit tests.</small> - </h1> -</div> - -<p>The <b>pathod.test</b> module is a light, flexible testing layer for HTTP clients. - It works by firing up a Pathod instance in a separate thread, letting you use - Pathod's full abilities to generate responses, and then query Pathod's internal - logs to establish what happened. All the mechanics of startup, shutdown, finding - free ports and so forth are taken care of for you. -</p> - -<p>The canonical docs can be accessed using pydoc: </p> - -<pre class="terminal">pydoc pathod.test</pre> - -<p> - The remainder of this page demonstrates some common interaction patterns using - <a href="http://nose.readthedocs.org/en/latest/">nose</a>. These examples are - also applicable with only minor modification to most commonly used Python testing - engines. -</p> - -<section> - <div class="page-header"> - <h1>Context Manager</h1> - </div> - - {% include "examples_context.html" %} -</section> - -<section> - <div class="page-header"> - <h1>One instance per test</h1> - </div> - - {% include "examples_setup.html" %} -</section> - -<section> - <div class="page-header"> - <h1>One instance per suite</h1> - </div> - - {% include "examples_setupall.html" %} -</section> -{% endblock %} diff --git a/pathod/templates/download.html b/pathod/templates/download.html deleted file mode 100644 index bd8950e8..00000000 --- a/pathod/templates/download.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<section> - <div class="page-header"> - <h1>pip</h1> - </div> - - <p>The easiest way to install pathod is to use pip:</p> - - <pre>pip install pathod</pre> - - <p> - This will automatically pull in all the dependencies, and you should be good to go. - </p> -</section> - -<section> - <div class="page-header"> - <h1>github</h1> - </div> - - <p>You can find the project source on GitHub:</p> - - <div style="margin-top: 20px; margin-bottom: 20px"> - <a class="btn btn-primary btn-large" href="https://github.com/mitmproxy/pathod">github.com/mitmproxy/pathod</a> - </div> - - <p>Please also use the <a href="https://github.com/mitmproxy/pathod/issues">github issue tracker</a> to report bugs.</p> -</section> - -<section> - <div class="page-header"> - <h1>tarball</h1> - </div> - - <div style="margin-top: 20px; margin-bottom: 20px"> - <a class="btn btn-primary btn-large" href="https://github.com/downloads/mitmproxy/pathod/pathod-{{version}}.tar.gz">pathod-{{version}}.tar.gz</a> - </div> -</section> -{% endblock %} diff --git a/pathod/templates/examples_context.html b/pathod/templates/examples_context.html deleted file mode 100644 index 20b02c39..00000000 --- a/pathod/templates/examples_context.html +++ /dev/null @@ -1,24 +0,0 @@ -<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span> -<span class="kn">from</span> <span class="nn">pathod</span> <span class="kn">import</span> <span class="n">test</span> - - -<span class="k">def</span> <span class="nf">test_simple</span><span class="p">():</span> - <span class="sd">"""</span> -<span class="sd"> Testing the requests module with</span> -<span class="sd"> a pathod context manager.</span> -<span class="sd"> """</span> - <span class="c"># Start pathod in a separate thread</span> - <span class="k">with</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span> <span class="k">as</span> <span class="n">d</span><span class="p">:</span> - <span class="c"># Get a URL for a pathod spec</span> - <span class="n">url</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">"200:b@100"</span><span class="p">)</span> - <span class="c"># ... and request it</span> - <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span> - - <span class="c"># Check the returned data</span> - <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> - <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span> - - <span class="c"># Check pathod's internal log</span> - <span class="n">log</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">"request"</span><span class="p">]</span> - <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">"method"</span><span class="p">]</span> <span class="o">==</span> <span class="s">"PUT"</span> -</pre></div> diff --git a/pathod/templates/examples_setup.html b/pathod/templates/examples_setup.html deleted file mode 100644 index b8419171..00000000 --- a/pathod/templates/examples_setup.html +++ /dev/null @@ -1,32 +0,0 @@ -<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span> -<span class="kn">from</span> <span class="nn">pathod</span> <span class="kn">import</span> <span class="n">test</span> - - -<span class="k">class</span> <span class="nc">Test</span><span class="p">:</span> - - <span class="sd">"""</span> -<span class="sd"> Testing the requests module with</span> -<span class="sd"> a pathod instance started for</span> -<span class="sd"> each test.</span> -<span class="sd"> """</span> - - <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">d</span> <span class="o">=</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span> - - <span class="k">def</span> <span class="nf">tearDown</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span> - - <span class="k">def</span> <span class="nf">test_simple</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="c"># Get a URL for a pathod spec</span> - <span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">"200:b@100"</span><span class="p">)</span> - <span class="c"># ... and request it</span> - <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span> - - <span class="c"># Check the returned data</span> - <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> - <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span> - - <span class="c"># Check pathod's internal log</span> - <span class="n">log</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">"request"</span><span class="p">]</span> - <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">"method"</span><span class="p">]</span> <span class="o">==</span> <span class="s">"PUT"</span> -</pre></div> diff --git a/pathod/templates/examples_setupall.html b/pathod/templates/examples_setupall.html deleted file mode 100644 index 3308f6cb..00000000 --- a/pathod/templates/examples_setupall.html +++ /dev/null @@ -1,40 +0,0 @@ -<div class="highlight"><pre><span class="kn">import</span> <span class="nn">requests</span> -<span class="kn">from</span> <span class="nn">pathod</span> <span class="kn">import</span> <span class="n">test</span> - - -<span class="k">class</span> <span class="nc">Test</span><span class="p">:</span> - - <span class="sd">"""</span> -<span class="sd"> Testing the requests module with</span> -<span class="sd"> a single pathod instance started</span> -<span class="sd"> for the test suite.</span> -<span class="sd"> """</span> - <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">setUpAll</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> - <span class="n">cls</span><span class="o">.</span><span class="n">d</span> <span class="o">=</span> <span class="n">test</span><span class="o">.</span><span class="n">Daemon</span><span class="p">()</span> - - <span class="nd">@classmethod</span> - <span class="k">def</span> <span class="nf">tearDownAll</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span> - <span class="n">cls</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span> - - <span class="k">def</span> <span class="nf">setUp</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="c"># Clear the pathod logs between tests</span> - <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">clear_log</span><span class="p">()</span> - - <span class="k">def</span> <span class="nf">test_simple</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="c"># Get a URL for a pathod spec</span> - <span class="n">url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">p</span><span class="p">(</span><span class="s">"200:b@100"</span><span class="p">)</span> - <span class="c"># ... and request it</span> - <span class="n">r</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">url</span><span class="p">)</span> - - <span class="c"># Check the returned data</span> - <span class="k">assert</span> <span class="n">r</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> - <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">r</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">==</span> <span class="mi">100</span> - - <span class="c"># Check pathod's internal log</span> - <span class="n">log</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">last_log</span><span class="p">()[</span><span class="s">"request"</span><span class="p">]</span> - <span class="k">assert</span> <span class="n">log</span><span class="p">[</span><span class="s">"method"</span><span class="p">]</span> <span class="o">==</span> <span class="s">"PUT"</span> - - <span class="k">def</span> <span class="nf">test_two</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> - <span class="k">assert</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">d</span><span class="o">.</span><span class="n">log</span><span class="p">()</span> -</pre></div> diff --git a/pathod/templates/frame.html b/pathod/templates/frame.html deleted file mode 100644 index 4223458d..00000000 --- a/pathod/templates/frame.html +++ /dev/null @@ -1,7 +0,0 @@ -{% extends "layout.html" %} {% block content %} -<div class="row"> - <div class="span12"> - {% block body %} {% endblock %} - </div> -</div> -{% endblock %} diff --git a/pathod/templates/index.html b/pathod/templates/index.html deleted file mode 100644 index a85a4040..00000000 --- a/pathod/templates/index.html +++ /dev/null @@ -1,60 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<div class="masthead"> - <div class="container"> - <h1>pathod: pathological HTTP</h1> - - <p>Crafted malice for tormenting HTTP clients and servers</p> - - <img src="/static/torture.png"> - </div> -</div> - -<div class="row"> - <div class="span6"> - <div> - <h2><a href="/docs/pathod">pathod</a></h2> - - <p>A pathological web daemon.</p> - - {% include "response_previewform.html" %} - <br> - </div> - </div> - - <div class="span6"> - <div> - <h2><a href="/docs/pathoc">pathoc</a></h2> - - <p>A perverse HTTP client.</p> - - {% include "request_previewform.html" %} - </div> - </div> -</div> - -<section> - <div class="page-header"> - <h1>Install</h1> - </div> - <div class="row"> - <div class="span6"> - <div> - <h2>pip</h2> - - <pre>pip install pathod</pre> - </div> - </div> - - <div class="span6"> - <div> - <h2>source</h2> - - <ul> - <li>Current release: <a href="http://mitmproxy.org/download/pathod-{{version}}.tar.gz">pathod {{version}}</a></li> - <li>GitHub: <a href="https://github.com/mitmproxy/pathod">github.com/mitmproxy/pathod</a></li> - </ul> - </div> - </div> - </div> -</section> -{% endblock %} diff --git a/pathod/templates/layout.html b/pathod/templates/layout.html deleted file mode 100644 index af2857b1..00000000 --- a/pathod/templates/layout.html +++ /dev/null @@ -1,75 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - -<head> - <meta charset="utf-8"> - <title>pathod</title> - <link href="/static/bootstrap.min.css" rel="stylesheet"> - <link href="/static/pathod.css" rel="stylesheet"> - <link href="/static/syntax.css" rel="stylesheet"> - <script src="/static/jquery-1.7.2.min.js"></script> - <script src="/static/jquery.scrollTo-min.js"></script> - <script src="/static/jquery.localscroll-min.js"></script> - <script src="/static/bootstrap.min.js"></script> - <meta name="viewport" content="width=device-width, initial-scale=1.0"> - <meta name="description" content=""> - <meta name="author" content=""> - <style type="text/css"> - body { - padding-top: 60px; - padding-bottom: 40px; - } - - </style> - <!-- Le HTML5 shim, for IE6-8 support of HTML5 elements --> - <!--[if lt IE 9]> - <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script> - <![endif]--> -</head> - -<body> - <div class="navbar navbar-fixed-top"> - <div class="navbar-inner"> - <div class="container"> - <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </a> - <a class="brand" href="/index.html">pathod</a> - <div class="nav-collapse"> - <ul class="nav"> - <li {% if section=="main" %} class="active" {% endif %}><a href="/">home</a></li> - {% if not noapi %} - <li {% if section=="log" %} class="active" {% endif %}><a href="/log">log</a></li> - {% endif %} - <li {% if section=="docs" %} class="active" {% endif %}><a href="/docs/pathod">docs</a></li> - <li {% if section=="about" %} class="active" {% endif %}><a href="/about">about</a></li> - </ul> - </div> - </div> - </div> - </div> - - <div class="container"> - {% block content %} {% endblock %} - <hr> - <footer> - <span>© Aldo Cortesi 2015</span> - <span class="pull-right">[served with pathod]</span> - </footer> - </div> -</body> -<script> - $(function() { - $.localScroll({ - duration: 300, - offset: { - top: -45 - } - }); - }); - -</script> - -</html> diff --git a/pathod/templates/log.html b/pathod/templates/log.html deleted file mode 100644 index b0484cb8..00000000 --- a/pathod/templates/log.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<form style="float: right" method="POST" action="/log/clear"> - <button type="submit" class="btn">clear</button> -</form> - -<h1>Logs</h1> -<hr> - -<table class="table table-striped table-condensed"> - <thead> - <tr> - <th>id</th> - <th>method</th> - <th>path</th> - </tr> - </thead> - <tbody> - {% for i in log %} - <tr> - {% if i["type"] == 'error' %} - <td colspan="3">ERROR: {{ i["msg"] }}</td> - {% else %} - <td>{{ i["id"] }}</td> - <td>{{ i["request"]["method"] }}</td> - <td><a href="/log/{{ i[" id "] }}">{{ i["request"]["path"] }}</a></td> - {% endif %} - </tr> - {% endfor %} - </tbody> -</table> -{% endblock %} diff --git a/pathod/templates/onelog.html b/pathod/templates/onelog.html deleted file mode 100644 index c222ad60..00000000 --- a/pathod/templates/onelog.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<h2>Log entry {{ lid }}</h2> -<hr> - -<pre> - {{ alog }} -</pre> -{% endblock %} diff --git a/pathod/templates/request_preview.html b/pathod/templates/request_preview.html deleted file mode 100644 index 25d73679..00000000 --- a/pathod/templates/request_preview.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<div class="page-header"> - <h1>pathoc preview</h1> -</div> - -<div style="margin-bottom: 20px" class="row"> - <div class="span2 header"> - Specification: - </div> - <div class="span10"> - {% include "request_previewform.html" %} - </div> -</div> - -{% if syntaxerror %} -<div class="row"> - <div class="span2 header"> - Error: - </div> - <div class="span10"> - <p style="color: #ff0000">{{ syntaxerror }}</p> - <pre>{{ marked }}</pre> - </div> -</div> -{% elif error %} -<div class="row"> - <div class="span2 header"> - Error: - </div> - <div class="span10"> - <p style="color: #ff0000">{{ error }}</p> - </div> -</div> -{% else %} -<div class="row"> - <div class="span2 header"> - Request: - </div> - <div class="span10"> - <pre>{{ output }}</pre> - <p>Note: pauses are skipped when generating previews!</p> - </div> -</div> -{% endif %} {% endblock %} diff --git a/pathod/templates/request_previewform.html b/pathod/templates/request_previewform.html deleted file mode 100644 index 91b5598a..00000000 --- a/pathod/templates/request_previewform.html +++ /dev/null @@ -1,53 +0,0 @@ -<form style="margin-bottom: 0" class="form-inline" method="GET" action="/request_preview"> - <input style="width: 18em" id="spec" name="spec" class="input-medium" value="{{spec}}" - placeholder="method:path:[features]"> - <input type="submit" class="btn" value="preview"> -</form> - -<a class="innerlink" data-toggle="collapse" data-target="#requestexamples">examples</a> - -<div id="requestexamples" class="collapse"> - <p> - Check out the <a href="/docs/language">complete language docs</a>. Here are - some examples to get you started: - </p> - - <table class="table table-bordered"> - <tbody> - <tr> - <td><a href="/request_preview?spec=get:/">get:/</a></td> - <td>Get path /</td> - </tr> - <tr> - <td><a href="/request_preview?spec=get:/:b@100">get:/:b@100</a></td> - <td>100 random bytes as the body</td> - </tr> - <tr> - <td><a href='/request_preview?spec=get:/:h"Etag"="';drop table browsers;"'>get:/:h"Etag"="';drop table browsers;"</a></td> - <td>Add a header</td> - </tr> - <tr> - <td><a href='/request_preview?spec=get:/:u"';drop table browsers;"'>get:/:u"';drop table browsers;"</a></td> - <td>Add a User-Agent header</td> - </tr> - <tr> - <td><a href="/request_preview?spec=get:/:b@100:dr">get:/:b@100:dr</a></td> - <td>Drop the connection randomly</td> - </tr> - <tr> - <td> - <a href="/request_preview?spec="></a> - </td> - <td></td> - </tr> - <tr> - <td><a href="/request_preview?spec=get:/:b@100,ascii:ir,@1">get:/:b@100,ascii:ir,@1</a></td> - <td>100 ASCII bytes as the body, and randomly inject a random byte</td> - </tr> - <tr> - <td><a href="/request_preview?spec=ws:/">ws:/</a></td> - <td>Initiate a websocket handshake.</td> - </tr> - </tbody> - </table> -</div> diff --git a/pathod/templates/response_preview.html b/pathod/templates/response_preview.html deleted file mode 100644 index bbce6d6c..00000000 --- a/pathod/templates/response_preview.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "frame.html" %} {% block body %} -<div class="page-header"> - <h1>pathod preview</h1> -</div> - -<div style="margin-bottom: 20px" class="row"> - <div class="span2 header"> - Specification: - </div> - <div class="span10"> - {% include "response_previewform.html" %} - </div> -</div> - -{% if syntaxerror %} -<div class="row"> - <div class="span2 header"> - Error: - </div> - <div class="span10"> - <p style="color: #ff0000">{{ syntaxerror }}</p> - <pre>{{ marked }}</pre> - </div> -</div> -{% elif error %} -<div class="row"> - <div class="span2 header"> - Error: - </div> - <div class="span10"> - <p style="color: #ff0000">{{ error }}</p> - </div> -</div> -{% else %} -<div class="row"> - <div class="span2 header"> - Response: - </div> - <div class="span10"> - <pre>{{ output }}</pre> - <p>Note: pauses are skipped when generating previews!</p> - </div> -</div> -{% endif %} {% endblock %} diff --git a/pathod/templates/response_previewform.html b/pathod/templates/response_previewform.html deleted file mode 100644 index d46043f3..00000000 --- a/pathod/templates/response_previewform.html +++ /dev/null @@ -1,87 +0,0 @@ -<form style="margin-bottom: 0" class="form-inline" method="GET" action="/response_preview"> - <input style="width: 18em" id="spec" name="spec" class="input-medium" value="{{spec}}" - placeholder="code:[features]"> - <input type="submit" class="btn" value="preview"> - {% if not nocraft %} - <a href="#" id="submitspec" class="btn">go</a> - {% endif %} -</form> - -<a class="innerlink" data-toggle="collapse" data-target="#responseexamples">examples</a> - -<div id="responseexamples" class="collapse"> - <p> - Check out the <a href="/docs/language">complete language docs</a>. Here are - some examples to get you started: - </p> - - <table class="table table-bordered"> - <tbody> - <tr> - <td><a href="/response_preview?spec=200">200</a></td> - <td>A basic HTTP 200 response.</td> - </tr> - <tr> - <td><a href="/response_preview?spec=200:r">200:r</a></td> - <td>A basic HTTP 200 response with no Content-Length header. This will - hang. - </td> - </tr> - <tr> - <td><a href="/response_preview?spec=200:da">200:da</a></td> - <td>Server-side disconnect after all content has been sent.</td> - </tr> - <tr> - <td><a href="/response_preview?spec=200:b@100">200:b@100</a></td> - <td> - 100 random bytes as the body. A Content-Lenght header is added, so the disconnect - is no longer needed. - </td> - </tr> - <tr> - <td><a href='/response_preview?spec=200:b@100:h"Server"="';drop table servers;"'>200:b@100:h"Etag"="';drop table servers;"</a></td> - <td>Add a Server header</td> - </tr> - <tr> - <td><a href="/response_preview?spec=200:b@100:dr">200:b@100:dr</a></td> - <td>Drop the connection randomly</td> - </tr> - <tr> - <td><a href="/response_preview?spec=200:b@100,ascii:ir,@1">200:b@100,ascii:ir,@1</a></td> - <td>100 ASCII bytes as the body, and randomly inject a random byte</td> - </tr> - <tr> - <td><a href='/response_preview?spec=200:b@1k:c"text/json"'>200:b@1k:c"text/json"</a></td> - <td>1k of random bytes, with a text/json content type</td> - </tr> - <tr> - <td><a href='/response_preview?spec=200:b@1k:p50,120'>200:b@1k:p50,120</a></td> - <td>1k of random bytes, pause for 120 seconds after 50 bytes</td> - </tr> - <tr> - <td><a href='/response_preview?spec=200:b@1k:pr,f'>200:b@1k:pr,f</a></td> - <td>1k of random bytes, but hang forever at a random location</td> - </tr> - <tr> - <td> - <a href="/response_preview?spec=200:b@100:h@1k,ascii_letters='foo'">200:b@100:h@1k,ascii_letters='foo'</a> - </td> - <td> - 100 ASCII bytes as the body, randomly generated 100k header name, with the value - 'foo'. - </td> - </tr> - </tbody> - </table> -</div> - -{% if not nocraft %} -<script> - $(function() { - $("#submitspec").click(function() { - document.location = "{{craftanchor}}" + $("#spec").val() - }); - }); - -</script> -{% endif %} diff --git a/pathod/utils.py b/pathod/utils.py index 3276198a..9b220e9a 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -17,15 +17,6 @@ class MemBool(object): return bool(v) -def parse_anchor_spec(s): - """ - Return a tuple, or None on error. - """ - if "=" not in s: - return None - return tuple(s.split("=", 1)) - - data = netlib.utils.Data(__name__) @@ -75,7 +75,7 @@ setup( "passlib>=1.6.5, <1.7", "pyasn1>=0.1.9, <0.2", "pyOpenSSL>=16.0, <17.0", - "pyparsing>=2.0, <2.1", # 2.1.1 breaks our binaries, see https://sourceforge.net/p/pyparsing/bugs/93/ + "pyparsing>=2.1.3, <2.2", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <2.10", "six>=1.10, <1.11", @@ -97,7 +97,6 @@ setup( "ipaddress>=1.0.15, <1.1", ], 'dev': [ - "coveralls>=1.1, <1.2", "mock>=2.0, <2.1", "flake8>=2.5.4, <3", "pytest>=2.8.7, <2.10", diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index 9142bdad..48d6c307 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,3 +1,5 @@ +import json + from mitmproxy.exceptions import ContentViewException from netlib.http import Headers from netlib.odict import ODict @@ -274,3 +276,9 @@ if cv.ViewProtobuf.is_available(): def test_get_by_shortcut(): assert cv.get_by_shortcut("h") + + +def test_pretty_json(): + s = json.dumps({"foo": 1}) + assert cv.pretty_json(s) + assert not cv.pretty_json("moo") diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils.py index c01b5f2a..79b72eec 100644 --- a/test/mitmproxy/test_utils.py +++ b/test/mitmproxy/test_utils.py @@ -1,29 +1,14 @@ -import json from mitmproxy import utils from . import tutils utils.CERT_SLEEP_TIME = 0 -def test_format_timestamp(): - assert utils.format_timestamp(utils.timestamp()) - - -def test_format_timestamp_with_milli(): - assert utils.format_timestamp_with_milli(utils.timestamp()) - - def test_pkg_data(): assert utils.pkg_data.path("console") tutils.raises("does not exist", utils.pkg_data.path, "nonexistent") -def test_pretty_json(): - s = json.dumps({"foo": 1}) - assert utils.pretty_json(s) - assert not utils.pretty_json("moo") - - def test_LRUCache(): cache = utils.LRUCache(2) diff --git a/test/netlib/test_human.py b/test/netlib/test_human.py index 2a5c2a85..bb97dc54 100644 --- a/test/netlib/test_human.py +++ b/test/netlib/test_human.py @@ -1,6 +1,15 @@ +import time from netlib import human, tutils +def test_format_timestamp(): + assert human.format_timestamp(time.time()) + + +def test_format_timestamp_with_milli(): + assert human.format_timestamp_with_milli(time.time()) + + def test_parse_size(): assert human.parse_size("0") == 0 assert human.parse_size("0b") == 0 diff --git a/test/pathod/test_language_base.py b/test/pathod/test_language_base.py index 075dc2b8..7c7d8cf9 100644 --- a/test/pathod/test_language_base.py +++ b/test/pathod/test_language_base.py @@ -55,8 +55,15 @@ class TestTokValueLiteral: v = base.TokValueLiteral("f\x00oo") assert v.spec() == repr(v) == r"'f\x00oo'" - v = base.TokValueLiteral("\"") - assert v.spec() == repr(v) == '\'"\'' + v = base.TokValueLiteral('"') + assert v.spec() == repr(v) == """ '"' """.strip() + + # While pyparsing has a escChar argument for QuotedString, + # escChar only performs scapes single-character escapes and does not work for e.g. r"\x02". + # Thus, we cannot use that option, which means we cannot have single quotes in strings. + # To fix this, we represent single quotes as r"\x07". + v = base.TokValueLiteral("'") + assert v.spec() == r"'\x27'" def roundtrip(self, spec): e = base.TokValueLiteral.expr() @@ -311,7 +318,7 @@ def test_options_or_value(): def test_integer(): e = base.Integer.expr() v = e.parseString("200")[0] - assert v.string() == "200" + assert v.string() == b"200" assert v.spec() == "200" assert v.freeze({}).value == v.value diff --git a/test/pathod/test_language_http.py b/test/pathod/test_language_http.py index d1870a63..18059e3a 100644 --- a/test/pathod/test_language_http.py +++ b/test/pathod/test_language_http.py @@ -1,15 +1,15 @@ -from six.moves import cStringIO as StringIO +from six import BytesIO from pathod import language from pathod.language import http, base import tutils def parse_request(s): - return language.parse_pathoc(s).next() + return next(language.parse_pathoc(s)) def test_make_error_response(): - d = StringIO() + d = BytesIO() s = http.make_error_response("foo") language.serve(s, d, {}) @@ -24,17 +24,17 @@ class TestRequest: def test_simple(self): r = parse_request('GET:"/foo"') - assert r.method.string() == "GET" - assert r.path.string() == "/foo" + assert r.method.string() == b"GET" + assert r.path.string() == b"/foo" r = parse_request('GET:/foo') - assert r.path.string() == "/foo" + assert r.path.string() == b"/foo" r = parse_request('GET:@1k') assert len(r.path.string()) == 1024 def test_multiple(self): r = list(language.parse_pathoc("GET:/ PUT:/")) - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" assert len(r) == 2 l = """ @@ -54,8 +54,8 @@ class TestRequest: """ r = list(language.parse_pathoc(l)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" l = """ get:"http://localhost:9999/p/200":ir,@1 @@ -63,8 +63,8 @@ class TestRequest: """ r = list(language.parse_pathoc(l)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "GET" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"GET" def test_nested_response(self): l = "get:/p:s'200'" @@ -75,7 +75,7 @@ class TestRequest: assert r[0].values({}) def test_render(self): - s = StringIO() + s = BytesIO() r = parse_request("GET:'/foo'") assert language.serve( r, @@ -90,8 +90,8 @@ class TestRequest: ir,@1 """ r = parse_request(l) - assert r.method.string() == "GET" - assert r.path.string() == "/foo" + assert r.method.string() == b"GET" + assert r.path.string() == b"/foo" assert r.actions l = """ @@ -106,8 +106,8 @@ class TestRequest: ir,@1 """ r = parse_request(l) - assert r.method.string() == "GET" - assert r.path.string().endswith("bar") + assert r.method.string() == b"GET" + assert r.path.string().endswith(b"bar") assert r.actions def test_spec(self): @@ -128,66 +128,66 @@ class TestRequest: def test_websocket(self): r = parse_request('ws:/path/') res = r.resolve(language.Settings()) - assert res.method.string().lower() == "get" - assert res.tok(http.Path).value.val == "/path/" - assert res.tok(http.Method).value.val.lower() == "get" - assert http.get_header("Upgrade", res.headers).value.val == "websocket" + assert res.method.string().lower() == b"get" + assert res.tok(http.Path).value.val == b"/path/" + assert res.tok(http.Method).value.val.lower() == b"get" + assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" r = parse_request('ws:put:/path/') res = r.resolve(language.Settings()) - assert r.method.string().lower() == "put" - assert res.tok(http.Path).value.val == "/path/" - assert res.tok(http.Method).value.val.lower() == "put" - assert http.get_header("Upgrade", res.headers).value.val == "websocket" + assert r.method.string().lower() == b"put" + assert res.tok(http.Path).value.val == b"/path/" + assert res.tok(http.Method).value.val.lower() == b"put" + assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" class TestResponse: def dummy_response(self): - return language.parse_pathod("400'msg'").next() + return next(language.parse_pathod("400'msg'")) def test_response(self): - r = language.parse_pathod("400:m'msg'").next() - assert r.status_code.string() == "400" - assert r.reason.string() == "msg" + r = next(language.parse_pathod("400:m'msg'")) + assert r.status_code.string() == b"400" + assert r.reason.string() == b"msg" - r = language.parse_pathod("400:m'msg':b@100b").next() - assert r.reason.string() == "msg" + r = next(language.parse_pathod("400:m'msg':b@100b")) + assert r.reason.string() == b"msg" assert r.body.values({}) assert str(r) - r = language.parse_pathod("200").next() - assert r.status_code.string() == "200" + r = next(language.parse_pathod("200")) + assert r.status_code.string() == b"200" assert not r.reason - assert "OK" in [i[:] for i in r.preamble({})] + assert b"OK" in [i[:] for i in r.preamble({})] def test_render(self): - s = StringIO() - r = language.parse_pathod("400:m'msg'").next() + s = BytesIO() + r = next(language.parse_pathod("400:m'msg'")) assert language.serve(r, s, {}) - r = language.parse_pathod("400:p0,100:dr").next() + r = next(language.parse_pathod("400:p0,100:dr")) assert "p0" in r.spec() s = r.preview_safe() assert "p0" not in s.spec() def test_raw(self): - s = StringIO() - r = language.parse_pathod("400:b'foo'").next() + s = BytesIO() + r = next(language.parse_pathod("400:b'foo'")) language.serve(r, s, {}) v = s.getvalue() - assert "Content-Length" in v + assert b"Content-Length" in v - s = StringIO() - r = language.parse_pathod("400:b'foo':r").next() + s = BytesIO() + r = next(language.parse_pathod("400:b'foo':r")) language.serve(r, s, {}) v = s.getvalue() - assert "Content-Length" not in v + assert b"Content-Length" not in v def test_length(self): def testlen(x): - s = StringIO() - x = x.next() + s = BytesIO() + x = next(x) language.serve(x, s, language.Settings()) assert x.length(language.Settings()) == len(s.getvalue()) testlen(language.parse_pathod("400:m'msg':r")) @@ -196,8 +196,8 @@ class TestResponse: def test_maximum_length(self): def testlen(x): - x = x.next() - s = StringIO() + x = next(x) + s = BytesIO() m = x.maximum_length({}) language.serve(x, s, {}) assert m >= len(s.getvalue()) @@ -225,19 +225,19 @@ class TestResponse: tutils.raises("ascii", language.parse_pathod, "foo:b\xf0") def test_parse_header(self): - r = language.parse_pathod('400:h"foo"="bar"').next() - assert http.get_header("foo", r.headers) + r = next(language.parse_pathod('400:h"foo"="bar"')) + assert http.get_header(b"foo", r.headers) def test_parse_pause_before(self): - r = language.parse_pathod("400:p0,10").next() + r = next(language.parse_pathod("400:p0,10")) assert r.actions[0].spec() == "p0,10" def test_parse_pause_after(self): - r = language.parse_pathod("400:pa,10").next() + r = next(language.parse_pathod("400:pa,10")) assert r.actions[0].spec() == "pa,10" def test_parse_pause_random(self): - r = language.parse_pathod("400:pr,10").next() + r = next(language.parse_pathod("400:pr,10")) assert r.actions[0].spec() == "pr,10" def test_parse_stress(self): @@ -245,29 +245,29 @@ class TestResponse: # returns an int and a python 2.7 int on windows has 32bit precision. # Therefore, we should keep the body length < 2147483647 bytes in our # tests. - r = language.parse_pathod("400:b@1g").next() + r = next(language.parse_pathod("400:b@1g")) assert r.length({}) def test_spec(self): def rt(s): - s = language.parse_pathod(s).next().spec() - assert language.parse_pathod(s).next().spec() == s + s = next(language.parse_pathod(s)).spec() + assert next(language.parse_pathod(s)).spec() == s rt("400:b@100g") rt("400") rt("400:da") def test_websockets(self): - r = language.parse_pathod("ws").next() + r = next(language.parse_pathod("ws")) tutils.raises("no websocket key", r.resolve, language.Settings()) - res = r.resolve(language.Settings(websocket_key="foo")) - assert res.status_code.string() == "101" + res = r.resolve(language.Settings(websocket_key=b"foo")) + assert res.status_code.string() == b"101" def test_ctype_shortcut(): e = http.ShortcutContentType.expr() v = e.parseString("c'foo'")[0] - assert v.key.val == "Content-Type" - assert v.value.val == "foo" + assert v.key.val == b"Content-Type" + assert v.value.val == b"foo" s = v.spec() assert s == e.parseString(s)[0].spec() @@ -282,8 +282,8 @@ def test_ctype_shortcut(): def test_location_shortcut(): e = http.ShortcutLocation.expr() v = e.parseString("l'foo'")[0] - assert v.key.val == "Location" - assert v.value.val == "foo" + assert v.key.val == b"Location" + assert v.value.val == b"foo" s = v.spec() assert s == e.parseString(s)[0].spec() @@ -296,23 +296,23 @@ def test_location_shortcut(): def test_shortcuts(): - assert language.parse_pathod( - "400:c'foo'").next().headers[0].key.val == "Content-Type" - assert language.parse_pathod( - "400:l'foo'").next().headers[0].key.val == "Location" + assert next(language.parse_pathod( + "400:c'foo'")).headers[0].key.val == b"Content-Type" + assert next(language.parse_pathod( + "400:l'foo'")).headers[0].key.val == b"Location" - assert "Android" in tutils.render(parse_request("get:/:ua")) - assert "User-Agent" in tutils.render(parse_request("get:/:ua")) + assert b"Android" in tutils.render(parse_request("get:/:ua")) + assert b"User-Agent" in tutils.render(parse_request("get:/:ua")) def test_user_agent(): e = http.ShortcutUserAgent.expr() v = e.parseString("ua")[0] - assert "Android" in v.string() + assert b"Android" in v.string() e = http.ShortcutUserAgent.expr() v = e.parseString("u'a'")[0] - assert "Android" not in v.string() + assert b"Android" not in v.string() v = e.parseString("u@100'")[0] assert len(str(v.freeze({}).value)) > 100 @@ -324,7 +324,7 @@ def test_user_agent(): def test_nested_response(): e = http.NestedResponse.expr() v = e.parseString("s'200'")[0] - assert v.value.val == "200" + assert v.value.val == b"200" tutils.raises( language.ParseException, e.parseString, @@ -340,9 +340,7 @@ def test_nested_response(): def test_nested_response_freeze(): e = http.NestedResponse( base.TokValueLiteral( - "200:b'foo':i10,'\\x27'".encode( - "string_escape" - ) + r"200:b\'foo\':i10,\'\\x27\'" ) ) assert e.freeze({}) diff --git a/test/pathod/test_language_websocket.py b/test/pathod/test_language_websocket.py index 29155dba..58297141 100644 --- a/test/pathod/test_language_websocket.py +++ b/test/pathod/test_language_websocket.py @@ -6,7 +6,7 @@ import tutils def parse_request(s): - return language.parse_pathoc(s).next() + return next(language.parse_pathoc(s)) class TestWebsocketFrame: @@ -93,9 +93,9 @@ class TestWebsocketFrame: def test_rawbody(self): frm = self.fr("wf:mask:r'foo'") assert len(frm.payload) == 3 - assert frm.payload != "foo" + assert frm.payload != b"foo" - assert self.fr("wf:r'foo'").payload == "foo" + assert self.fr("wf:r'foo'").payload == b"foo" def test_construction_2(self): # Simple server frame @@ -109,7 +109,7 @@ class TestWebsocketFrame: assert frm.header.masking_key frm = self.fr("wf:b'foo':k'abcd'", is_client=True) assert frm.header.mask - assert frm.header.masking_key == 'abcd' + assert frm.header.masking_key == b'abcd' # Server frame, mask explicitly set frm = self.fr("wf:b'foo':mask") @@ -117,7 +117,7 @@ class TestWebsocketFrame: assert frm.header.masking_key frm = self.fr("wf:b'foo':k'abcd'") assert frm.header.mask - assert frm.header.masking_key == 'abcd' + assert frm.header.masking_key == b'abcd' # Client frame, mask explicitly unset frm = self.fr("wf:b'foo':-mask", is_client=True) @@ -128,7 +128,7 @@ class TestWebsocketFrame: assert not frm.header.mask # We're reading back a corrupted frame - the first 3 characters of the # mask is mis-interpreted as the payload - assert frm.payload == "abc" + assert frm.payload == b"abc" def test_knone(self): with tutils.raises("expected 4 bytes"): @@ -138,5 +138,5 @@ class TestWebsocketFrame: assert self.fr("wf:l3:b'foo'").header.payload_length == 3 frm = self.fr("wf:l2:b'foo'") assert frm.header.payload_length == 2 - assert frm.payload == "fo" + assert frm.payload == b"fo" tutils.raises("expected 1024 bytes", self.fr, "wf:l1024:b'foo'") diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 0bbad6c2..ed4ef49f 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -145,14 +145,14 @@ class CommonTests(tutils.DaemonTests): def test_invalid_first_line(self): c = tcp.TCPClient(("localhost", self.d.port)) - c.connect() - if self.ssl: - c.convert_to_ssl() - c.wfile.write("foo\n\n\n") - c.wfile.flush() - l = self.d.last_log() - assert l["type"] == "error" - assert "foo" in l["msg"] + with c.connect(): + if self.ssl: + c.convert_to_ssl() + c.wfile.write("foo\n\n\n") + c.wfile.flush() + l = self.d.last_log() + assert l["type"] == "error" + assert "foo" in l["msg"] def test_invalid_content_length(self): tutils.raises( @@ -238,12 +238,12 @@ class TestDaemonSSL(CommonTests): c = tcp.TCPClient(("localhost", self.d.port)) c.rbufsize = 0 c.wbufsize = 0 - c.connect() - c.wfile.write("\0\0\0\0") - tutils.raises(TlsException, c.convert_to_ssl) - l = self.d.last_log() - assert l["type"] == "error" - assert "SSL" in l["msg"] + with c.connect(): + c.wfile.write("\0\0\0\0") + tutils.raises(TlsException, c.convert_to_ssl) + l = self.d.last_log() + assert l["type"] == "error" + assert "SSL" in l["msg"] def test_ssl_cipher(self): r, _ = self.pathoc([r"get:/p/202"]) diff --git a/test/pathod/test_pathod_cmdline.py b/test/pathod/test_pathod_cmdline.py index 3c0918ef..18d54c82 100644 --- a/test/pathod/test_pathod_cmdline.py +++ b/test/pathod/test_pathod_cmdline.py @@ -3,6 +3,11 @@ import tutils import mock +def test_parse_anchor_spec(): + assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200") + assert cmdline.parse_anchor_spec("foo") is None + + @mock.patch("argparse.ArgumentParser.error") def test_pathod(perror): assert cmdline.args_pathod(["pathod"]) diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py index a46a523a..2bb82fe7 100644 --- a/test/pathod/test_utils.py +++ b/test/pathod/test_utils.py @@ -11,10 +11,5 @@ def test_membool(): assert m.v == 2 -def test_parse_anchor_spec(): - assert utils.parse_anchor_spec("foo=200") == ("foo", "200") - assert utils.parse_anchor_spec("foo") is None - - def test_data_path(): tutils.raises(ValueError, utils.data.path, "nonexistent") diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index 56cd2002..bf5e3165 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -3,6 +3,7 @@ import re import shutil import requests from six.moves import cStringIO as StringIO +from six import BytesIO import urllib from netlib import tcp @@ -147,6 +148,6 @@ test_data = utils.Data(__name__) def render(r, settings=language.Settings()): r = r.resolve(settings) - s = StringIO() + s = BytesIO() assert language.serve(r, s, settings) return s.getvalue() @@ -8,7 +8,7 @@ deps = -rrequirements.txt commands = py.test -n 8 --timeout 60 ./test [testenv:py35] -commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py +commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test_language_websocket.py [testenv:lint] deps = flake8 diff --git a/web/src/js/ducks/utils/view.js b/web/src/js/ducks/utils/view.js index ff6f4c12..fa23efcd 100644 --- a/web/src/js/ducks/utils/view.js +++ b/web/src/js/ducks/utils/view.js @@ -43,12 +43,8 @@ const sortedRemove = (list, sortFn, item) => { } export function sortedIndexOf(list, value, sortFn) { - if (sortFn === false){ - let i = 0 - while (i < list.length && list[i].id !== value.id){ - i++ - } - return i + if (!sortFn) { + sortFn = x => 0 // This triggers the linear search for flows that have the same sort value. } let low = 0, @@ -57,7 +53,7 @@ export function sortedIndexOf(list, value, sortFn) { mid; while (low < high) { mid = (low + high) >>> 1; - if ((sortFn(list[mid]) < val) ) { + if (sortFn(list[mid]) < val) { low = mid + 1 } else { high = mid @@ -125,4 +121,4 @@ export function updateViewFilter(list, filterFn = defaultFilterFn, sortFn = defa filtered.indexOf = x => sortedIndexOf(filtered, x, sortFn) return filtered -}
\ No newline at end of file +} |