Merge branch 'master' of github.com:mitmproxy/mitmproxy

17 files changed, 338 insertions, 65 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 86c41b13..e0fb71b6 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,10 @@
+29 Dec 2014: mitmproxy 0.11.2:
+
+    * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
+      .mitmproxy directory.
+    * Better handling of servers that reject connections that are not SNI.
+    * Many other small bugfixes and improvements.
+
 
 15 November 2014: mitmproxy 0.11.1:
 
diff --git a/MANIFEST.in b/MANIFEST.in
index cc048b61..3578d855 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,4 +5,6 @@ recursive-include examples *
 recursive-include doc *
 recursive-include test *
 recursive-include libmproxy *
-recursive-exclude * *.pyc *.pyo *.swo *.swp
\ No newline at end of file
+recursive-exclude * *.pyc *.pyo *.swo *.swp
+recursive-exclude netlib *
+recursive-exclude libpathod *
diff --git a/doc-src/_nav.html b/doc-src/_nav.html
index 0ae0fa67..6c3afbe1 100644
--- a/doc-src/_nav.html
+++ b/doc-src/_nav.html
@@ -7,6 +7,7 @@
     <li class="nav-header">Tools</li>
         $!nav("mitmproxy.html", this, state)!$
         $!nav("mitmdump.html", this, state)!$
+        $!nav("config.html", this, state)!$
 
     <li class="nav-header">Features</li>
         $!nav("anticache.html", this, state)!$
diff --git a/doc-src/config.html b/doc-src/config.html
new file mode 100644
index 00000000..46688575
--- /dev/null
+++ b/doc-src/config.html
@@ -0,0 +1,86 @@
+
+
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory. 
+
+<table class="table">
+	<tbody>
+		<tr>
+			<th>mitmproxy.conf</th> 
+			<td>Settings for the <b>mitmproxy</b>. This file can contain any options supported by mitmproxy.</td>
+		</tr>
+		<tr>
+			<th>mitmdump.conf</th> 
+			<td>Settings for the <b>mitmdump</b>. This file can contain any options supported by mitmdump.</td>
+		</tr>
+		<tr>
+			<th>common.conf</th>
+
+			<td>Settings shared between all command-line tools. Settings in
+			this file are over-ridden by those in the tool-specific
+			files. Only options shared by mitmproxy and mitmdump should be used in this file. </td>
+		</tr>
+	</tbody>
+</table>
+
+# Syntax
+
+## Comments
+
+<pre>
+# this is a comment
+; this is also a comment (.ini style)
+--- and this is a comment too (yaml style)
+</pre>
+
+## Key/Value pairs
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+<pre>
+name = value   # (.ini style) 
+name: value    # (yaml style)
+--name value   # (command-line option style)
+
+fruit = [apple, orange, lemon]
+indexes = [1, 12, 35 , 40]
+</pre>
+
+## Flags
+
+These are boolean options that take no value but true/false.
+
+<pre>
+name = true    # (.ini style)
+name
+--name 	 	   # (command-line option style)
+</pre>
+
+# Options
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the __--help__ option on each of the tools. Be
+careful to only specify common options in the __common.conf__ file -
+unsupported options in this file will be detected as an error on startup.
+
+# Examples
+
+## common.conf
+
+Note that __port__ is an option supported by all tools. 
+
+<pre class="code">
+port = 8080
+</pre>
+
+
+## mitmproxy.conf
+
+<pre class="code">
+palette = light
+</pre>
+
+
diff --git a/doc-src/index.py b/doc-src/index.py
index 9eb0ea2b..753f90a5 100644
--- a/doc-src/index.py
+++ b/doc-src/index.py
@@ -4,10 +4,13 @@ import datetime
 import countershape
 from countershape import Page, Directory, markup, model
 import countershape.template
-sys.path.insert(0, "..")
+
+MITMPROXY_SRC = os.path.abspath(
+    os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
+)
+sys.path.insert(0, MITMPROXY_SRC)
 from libmproxy import filt, version
 
-MITMPROXY_SRC = os.environ.get("MITMPROXY_SRC", os.path.abspath(".."))
 ns.VERSION = version.VERSION
 
 if ns.options.website:
@@ -57,11 +60,13 @@ ns.navbar = countershape.template.File(None, "_nav.html")
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
-    Page("mitmproxy.html", "mitmproxy"),
-    Page("mitmdump.html", "mitmdump"),
     Page("howmitmproxy.html", "How mitmproxy works"),
     Page("modes.html", "Modes of Operation"),
 
+    Page("mitmproxy.html", "mitmproxy"),
+    Page("mitmdump.html", "mitmdump"),
+    Page("config.html", "configuration"),
+
     Page("ssl.html", "Overview"),
     Directory("certinstall"),
     Directory("scripting"),
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index ec03d63e..f5c66caa 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -195,6 +195,12 @@ def common_options(parser):
         version= "%(prog)s" + " " + version.VERSION
     )
     parser.add_argument(
+        '--shortversion',
+        action= 'version',
+        help = "show program's short version number and exit",
+        version = version.VERSION
+    )
+    parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
 
@@ -451,9 +457,9 @@ def common_options(parser):
         "--replay-ignore-payload-param",
         action="append", dest="replay_ignore_payload_params", type=str,
         help="""
-            Request's payload parameters (application/x-www-form-urlencoded) to 
-            be ignored while searching for a saved flow to replay. 
-            Can be passed multiple times. 
+            Request's payload parameters (application/x-www-form-urlencoded) to
+            be ignored while searching for a saved flow to replay.
+            Can be passed multiple times.
         """
     )
 
diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py
index 4023fae2..f0aecc15 100644
--- a/libmproxy/onboarding/app.py
+++ b/libmproxy/onboarding/app.py
@@ -1,29 +1,80 @@
 from __future__ import absolute_import
-import flask
 import os
+import tornado.web
+import tornado.wsgi
+import tornado.template
+
+from .. import utils
 from ..proxy import config
 
-mapp = flask.Flask(__name__)
-mapp.debug = True
+
+loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))
+
+
+class Adapter(tornado.wsgi.WSGIAdapter):
+    # Tornado doesn't make the WSGI environment available to pages, so this
+    # hideous monkey patch is the easiest way to get to the mitmproxy.master
+    # variable.
+    def __init__(self, application):
+        self._application = application
+
+    def application(self, request):
+        request.master = self.environ["mitmproxy.master"]
+        return self._application(request)
+
+    def __call__(self, environ, start_response):
+        self.environ = environ
+        return tornado.wsgi.WSGIAdapter.__call__(
+            self,
+            environ,
+            start_response
+        )
 
 
-def master():
-    return flask.request.environ["mitmproxy.master"]
+class Index(tornado.web.RequestHandler):
+    def get(self):
+        t = loader.load("index.html")
+        self.write(t.generate())
 
 
-@mapp.route("/")
-def index():
-    return flask.render_template("index.html", section="home")
+class PEM(tornado.web.RequestHandler):
+    def get(self):
+        p = os.path.join(
+            self.request.master.server.config.cadir,
+            config.CONF_BASENAME + "-ca-cert.pem"
+        )
+        self.set_header(
+            "Content-Type", "application/x-x509-ca-cert"
+        )
+        self.write(open(p, "rb").read())
 
 
-@mapp.route("/cert/pem")
-def certs_pem():
-    p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem")
-    return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert')
+class P12(tornado.web.RequestHandler):
+    def get(self):
+        p = os.path.join(
+            self.request.master.server.config.cadir,
+            config.CONF_BASENAME + "-ca-cert.p12"
+        )
+        self.set_header(
+            "Content-Type", "application/x-pkcs12"
+        )
+        self.write(open(p, "rb").read())
 
 
-@mapp.route("/cert/p12")
-def certs_p12():
-    p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12")
-    return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12')
+application = tornado.web.Application(
+    [
+        (r"/", Index),
+        (r"/cert/pem", PEM),
+        (r"/cert/p12", P12),
+        (
+            r"/static/(.*)",
+            tornado.web.StaticFileHandler,
+            {
+                "path": utils.pkg_data.path("onboarding/static")
+            }
+        ),
+    ],
+    #debug=True
+)
+mapp = Adapter(application)
 
diff --git a/libmproxy/onboarding/templates/frame.html b/libmproxy/onboarding/templates/frame.html
index b5c5c67c..f00e1a66 100644
--- a/libmproxy/onboarding/templates/frame.html
+++ b/libmproxy/onboarding/templates/frame.html
@@ -3,7 +3,7 @@
 <div class="row">
     <div class="span12">
         {% block body %}  
-        {% endblock %}  
+        {% end %}  
     </div>
 </div>
-{% endblock %}  
+{% end %}  
diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html
index 65fda5d2..1bcff1b8 100644
--- a/libmproxy/onboarding/templates/index.html
+++ b/libmproxy/onboarding/templates/index.html
@@ -32,4 +32,4 @@
     between mitmproxy installations.
 </div>
 
-{% endblock %}  
+{% end %}  
diff --git a/libmproxy/onboarding/templates/layout.html b/libmproxy/onboarding/templates/layout.html
index 622858be..8726a788 100644
--- a/libmproxy/onboarding/templates/layout.html
+++ b/libmproxy/onboarding/templates/layout.html
@@ -25,7 +25,7 @@
 
 <div class="container">
     {% block content %}  
-    {% endblock %}  
+    {% end %}  
 </div>
 
   </body>
diff --git a/release/mitmdump.spec b/release/mitmdump.spec
new file mode 100644
index 00000000..a84908f5
--- /dev/null
+++ b/release/mitmdump.spec
@@ -0,0 +1,32 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmdump'],
+             hiddenimports=[],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None,
+             cipher=block_cipher,
+          )
+a.datas = Tree(
+  "./libmproxy/onboarding/templates",
+  prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+  "./libmproxy/onboarding/static",
+  prefix="libmproxy/onboarding/static"
+)
+pyz = PYZ(a.pure,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='mitmdump',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )
diff --git a/release/mitmproxy.spec b/release/mitmproxy.spec
new file mode 100644
index 00000000..546c2899
--- /dev/null
+++ b/release/mitmproxy.spec
@@ -0,0 +1,32 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmproxy'],
+             hiddenimports=[],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None,
+             cipher=block_cipher,
+          )
+a.datas = Tree(
+  "./libmproxy/onboarding/templates",
+  prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+  "./libmproxy/onboarding/static",
+  prefix="libmproxy/onboarding/static"
+)
+pyz = PYZ(a.pure,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='mitmproxy',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )
diff --git a/release/mitmweb.spec b/release/mitmweb.spec
new file mode 100644
index 00000000..4e99da38
--- /dev/null
+++ b/release/mitmweb.spec
@@ -0,0 +1,40 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmweb'],
+             hiddenimports=[],
+             hookspath=None,
+             runtime_hooks=None,
+             excludes=None,
+             cipher=block_cipher,
+          )
+a.datas = Tree(
+  "./libmproxy/onboarding/templates",
+  prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+  "./libmproxy/onboarding/static",
+  prefix="libmproxy/onboarding/static"
+)
+a.datas += Tree(
+  "./libmproxy/web/templates",
+  prefix="libmproxy/web/templates"
+)
+a.datas += Tree(
+  "./libmproxy/web/static",
+  prefix="libmproxy/web/static"
+)
+pyz = PYZ(a.pure,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='mitmweb',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )
diff --git a/release/osx-binaries b/release/osx-binaries
index 9945e471..862ddd59 100755
--- a/release/osx-binaries
+++ b/release/osx-binaries
@@ -5,27 +5,45 @@
 # A few quirks to note, which should be re-checked every release:
 # - We require the latest development version of PyInstaller.
  
-# - PyInstaller has trouble detecting the zope.interfaces package. This is
-# required by Twisted, which for mysterious reasons is required by Urwid. The
-# answer is to touch the __init__.py file in the zope directory. On my system:
-#   touch /Library/Python/2.7/site-packages/zope/__init__.py
+# To run, first install netlib and mitmproxy, then run 
+#
+#	./release/osx-binaries
+#
+# From the top-level mitmproxy directory. 
 
-# To run, first install netlib and mitmproxy, then change into the pyinstaller
-# directory, and then run this script.
+usage ()
+{
+  echo 'Usage : ./release/osx-binaries /path/to/pyinstaller.py'
+  echo 'Run from the top-level mitmproxy directory'
+  exit
+}
 
-DST=/tmp/osx-mitmproxy
-MITMPROXY=~/mitmproxy/mitmproxy
-PYINST_CMD="./pyinstaller.py -F --clean"
+if [ "$1" = "" ]
+then
+    usage
+fi
 
-rm -rf $DST
-mkdir -p $DST
-rm -rf mitmproxy
-rm -rf mitmdump
 
-$PYINST_CMD $MITMPROXY/mitmproxy
-cp mitmproxy/dist/mitmproxy $DST
+TMPDIR=./tmp
+PYINST_CMD=$1" -F --clean"
 
-$PYINST_CMD $MITMPROXY/mitmdump
-cp mitmdump/dist/mitmdump $DST
+rm -f dist/*
+rm -rf $TMPDIR
 
-cshape $MITMPROXY/doc-src $DST/doc
+$PYINST_CMD ./release/mitmdump.spec
+./dist/mitmdump --version || exit 1
+
+$PYINST_CMD ./release/mitmproxy.spec
+./dist/mitmproxy --version || exit 1
+
+$PYINST_CMD ./release/mitmweb.spec
+./dist/mitmweb --version || exit 1
+
+DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1`
+mkdir -p $TMPDIR/$DST
+cp ./dist/mitmproxy $TMPDIR/$DST
+cp ./dist/mitmdump $TMPDIR/$DST
+cshape ./doc-src $TMPDIR/$DST/doc
+
+cd $TMPDIR
+tar -czvf $DST.tar.gz $DST
diff --git a/release/release-checklist b/release/release-checklist
index 31a1a48f..154103e9 100644
--- a/release/release-checklist
+++ b/release/release-checklist
@@ -1,5 +1,5 @@
 
-- Bump the version number: 
+- Check the version number:
 
     mitmproxy/libmproxy/version.py
     netlib/netlib/version.py
@@ -11,30 +11,24 @@
 - Run the test release, make sure the output is sensible
     ./release/test-release
 
-- Build sdist packages:
-    python ./setup.py sdist
-
-- Test the packages by installing in a virtualenv:
-    - cd dist
-    - tar -xzvf pkgfile.tgz
-    - virtualenv venv
-
 - Build the OSX binaries
     - Follow instructions in osxbinaries
-    - Package:
-        cp -r ./doc /tmp/osx-mitmproxy/
-        mv /tmp/osx-mitmproxy /tmp/osx-mitmproxy-VERSION
-        tar -czvf /tmp/osx-mitmproxy-VERSION.tar.gz /tmp/osx-mitmproxy-VERSION
-        mv /tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
+    - Move to download dir:
+        mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
 
 - Build the sources for each project:
     python ./setup.py sdist
     mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download
 
-
-- Adjust links on www.mitmproxy.org
+- Tag with the version number, and do: 
+    git push --tags
 
 - Upload to pypi for each project:
 
     python ./setup.py sdist upload
 
+- Now bump the version number to be ready for the next cycle: 
+
+    mitmproxy/libmproxy/version.py
+    netlib/netlib/version.py
+    pathod/libpathod/version.py
diff --git a/setup.py b/setup.py
index 79398a18..953df2bb 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,6 @@ deps = {
     "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
     "pyasn1>0.1.2",
     "pyOpenSSL>=0.14",
-    "Flask>=0.10.1",
     "tornado>=4.0.2",
     "configargparse>=0.9.3"
 }
diff --git a/test/tservers.py b/test/tservers.py
index 12154ba7..37929d1a 100644
--- a/test/tservers.py
+++ b/test/tservers.py
@@ -158,10 +158,10 @@ class HTTPProxTest(ProxTestBase):
         if self.ssl:
             p = libpathod.pathoc.Pathoc(("127.0.0.1", self.proxy.port), True)
             p.connect((APP_HOST, APP_PORT))
-            return p.request("get:'/%s'"%page)
+            return p.request("get:'%s'"%page)
         else:
             p = self.pathoc()
-            return p.request("get:'http://%s/%s'"%(APP_HOST, page))
+            return p.request("get:'http://%s%s'"%(APP_HOST, page))
 
 
 class TResolver: