update rtool

6 files changed, 337 insertions, 379 deletions

diff --git a/.env b/.env
index 97f38452..69ac3f05 100644
--- a/.env
+++ b/.env
@@ -1,5 +1,6 @@
-DIR="${0%/*}"
-if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
+DIR="$( dirname "${BASH_SOURCE[0]}" )"
+ACTIVATE_DIR="$(if [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate" ]; then
     echo "Activating mitmproxy virtualenv..."
-    source "$DIR/../venv.mitmproxy/bin/activate"
+    source "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate"
 fi
diff --git a/.gitignore b/.gitignore
index b3f4fc3e..c062fb3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,13 +1,9 @@
 .DS_Store
 MANIFEST
-/dist
-/tmp
-/venv
 *.py[cdo]
 *.swp
 *.swo
 
-/venv
-/release
 /build
-/pyinstallerdist
+/dist
+/mitmproxy_rtool.egg-info
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index de1b90b2..00000000
--- a/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-click>=4.1
-twine>=1.5.0
diff --git a/rtool b/rtool
deleted file mode 100755
index 7a4b0d58..00000000
--- a/rtool
+++ /dev/null
@@ -1,368 +0,0 @@
-#!/usr/bin/env python
-from __future__ import (
-    absolute_import, print_function, division
-)
-from os.path import join
-import contextlib
-import os.path
-import os
-import shutil
-import subprocess
-import glob
-import re
-import shlex
-import runpy
-import pprint
-
-import click
-
-# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
-# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
-if os.name == "nt":
-    VENV_BIN = "Scripts"
-else:
-    VENV_BIN = "bin"
-
-
-RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__)))
-DIST_DIR = join(RELEASE_DIR, "release")
-ROOT_DIR = join(RELEASE_DIR, "..")
-MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy")
-
-PYINSTALLER_URL =\
-    "https://github.com/pyinstaller/pyinstaller/archive/develop.zip"
-PYINSTALLER_CACHE = os.path.expanduser(
-    "~/Library/Application Support/pyinstaller"
-)
-PYINSTALLER_DIST = join(RELEASE_DIR, "pyinstallerdist")
-
-VENV_DIR = join(RELEASE_DIR, "venv")
-VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
-VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
-
-PROJECTS = {
-    "netlib": {
-        "tools": [],
-        "vfile": join(ROOT_DIR, "netlib/netlib/version.py"),
-        "version": None,
-        "dir": None
-    },
-    "pathod": {
-        "tools": ["pathod", "pathoc"],
-        "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"),
-        "version": None,
-        "dir": None
-    },
-    "mitmproxy": {
-        "tools": ["mitmproxy", "mitmdump", "mitmweb"],
-        "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"),
-        "version": None,
-        "dir": None
-    }
-}
-if os.name == "nt":
-    PROJECTS["mitmproxy"]["tools"].remove("mitmproxy")
-
-for project, settings in PROJECTS.items():
-    settings["version"] = runpy.run_path(settings["vfile"])["VERSION"]
-    settings["dir"] = join(ROOT_DIR, project)
-
-
-def proj(spec):
-    """
-        A small helper to iterate over filtered projects.
-    """
-    for k, v in PROJECTS.items():
-        if k not in spec:
-            continue
-        yield k, v
-
-
-@contextlib.contextmanager
-def empty_pythonpath():
-    """
-    Make sure that the regular python installation is not on the python path,
-    which would give us access to modules installed outside of our virtualenv.
-    """
-    pythonpath = os.environ["PYTHONPATH"]
-    os.environ["PYTHONPATH"] = ""
-    yield
-    os.environ["PYTHONPATH"] = pythonpath
-
-
-@contextlib.contextmanager
-def chdir(path):
-    old_dir = os.getcwd()
-    os.chdir(path)
-    yield
-    os.chdir(old_dir)
-
-
-@click.group(chain=True)
-def cli():
-    """
-    mitmproxy build tool
-    """
-    pass
-
-
-@cli.command("contributors")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-def contributors(projects):
-    """
-        Update CONTRIBUTORS.md
-    """
-    for project, conf in proj(projects):
-        with chdir(conf["dir"]):
-            print("Updating %s/CONTRIBUTORS..."%project)
-            contributors_data = subprocess.check_output(
-                shlex.split("git shortlog -n -s")
-            )
-            with open("CONTRIBUTORS", "w+") as f:
-                f.write(contributors_data)
-
-
-@cli.command("docs")
-def docs():
-    """
-    Render the docs
-    """
-    print("Rendering the docs...")
-    subprocess.check_call([
-        "cshape",
-        join(MITMPROXY_DIR, "doc-src"),
-        join(MITMPROXY_DIR, "doc")
-    ])
-
-
-@cli.command("set-version")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-@click.argument('version')
-def set_version(projects, version):
-    """
-    Update version information
-    """
-    print("Update versions...")
-    version = ", ".join(version.split("."))
-    for p, conf in proj(projects):
-        print("Update %s..." % conf["vfile"])
-        with open(conf["vfile"], "rb") as f:
-            content = f.read()
-        new_content = re.sub(
-            r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version,
-            content
-        )
-        with open(conf["vfile"], "wb") as f:
-            f.write(new_content)
-
-
-@cli.command("git")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-@click.argument('args', nargs=-1, required=True)
-def git(projects, args):
-    """
-    Run a git command on every project
-    """
-    args = ["git"] + list(args)
-    for project, conf in proj(projects):
-        print("%s> %s..." % (project, " ".join(args)))
-        subprocess.check_call(
-            args,
-            cwd=conf["dir"]
-        )
-
-
-@cli.command("sdist")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-def sdist(projects):
-    """
-    Build a source distribution
-    """
-    with empty_pythonpath():
-        print("Building release...")
-        if os.path.exists(DIST_DIR):
-            shutil.rmtree(DIST_DIR)
-        for project, conf in proj(projects):
-            print("Creating %s source distribution..." % project)
-            subprocess.check_call(
-                [
-                    "python", "./setup.py",
-                    "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar"
-                ],
-                cwd=conf["dir"]
-            )
-
-
-@cli.command("osxbin")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True,
-    type=click.Choice(PROJECTS.keys()),
-    default=PROJECTS.keys()
-)
-@click.pass_context
-def osxbin(ctx, projects):
-    if not os.path.exists(VENV_PYINSTALLER):
-        print("Instaling PyInstaller...")
-        subprocess.check_call(
-            [
-                VENV_PIP,
-                "install",
-                PYINSTALLER_URL
-            ]
-        )
-
-    shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True)
-    shutil.rmtree("./build", ignore_errors=True)
-    shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True)
-    for p, conf in proj(projects):
-        specs = glob.glob(os.path.join(conf["dir"], "release/*.spec"))
-        if specs:
-            for spec in specs:
-                subprocess.check_call(
-                    [
-                        VENV_PYINSTALLER,
-                        "--distpath", PYINSTALLER_DIST,
-                        spec
-                    ]
-                )
-            bins = os.listdir(PYINSTALLER_DIST)
-
-            base = os.path.join(DIST_DIR, "osx-" + p + "-" + conf["version"])
-            shutil.rmtree(base, ignore_errors=True)
-            os.makedirs(base)
-            for bin in bins:
-                bin = os.path.join(PYINSTALLER_DIST, bin)
-                subprocess.check_call([bin, "--version"])
-                shutil.move(bin, base)
-            subprocess.check_call(
-                [
-                    "tar",
-                    "-czvf",
-                    base + ".tgz",
-                    base
-                ]
-            )
-
-
-@cli.command("mkvenv")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-@click.pass_context
-def mkvenv(ctx, projects):
-    """
-    make a venv and test the source distribution
-    """
-    ctx.invoke(sdist)
-    with empty_pythonpath():
-        print("Creating virtualenv for test install...")
-        if os.path.exists(VENV_DIR):
-            shutil.rmtree(VENV_DIR)
-        subprocess.check_call(["virtualenv", "-q", VENV_DIR])
-
-        with chdir(DIST_DIR):
-            for project, conf in proj(projects):
-                print("Installing %s..." % project)
-                subprocess.check_call([VENV_PIP, "install", "-q", conf["dir"]])
-
-            print("Running binaries...")
-            for project, conf in proj(projects):
-                for tool in PROJECTS[project]["tools"]:
-                    tool = join(VENV_DIR, VENV_BIN, tool)
-                    print(tool)
-                    print(subprocess.check_output([tool, "--version"]))
-
-            print("Virtualenv available for further testing:")
-            print(
-                "source %s" % os.path.normpath(
-                    join(VENV_DIR, VENV_BIN, "activate")
-                )
-            )
-
-
-@cli.command("upload")
-@click.option('--username', prompt=True)
-@click.password_option(confirmation_prompt=False)
-@click.option('--repository', default="pypi")
-def upload_release(username, password, repository):
-    """
-    Upload source distributions to PyPI
-    """
-    print("Uploading distributions...")
-    subprocess.check_call([
-        "twine",
-        "upload",
-        "-u", username,
-        "-p", password,
-        "-r", repository,
-        "%s/*" % DIST_DIR
-    ])
-
-
-# TODO: Fully automate build process.
-# This wizard is missing OSX builds and updating mitmproxy.org.
-@cli.command("wizard")
-@click.option('--version', prompt=True)
-@click.option('--username', prompt="PyPI Username")
-@click.password_option(confirmation_prompt=False, prompt="PyPI Password")
-@click.option('--repository', default="pypi")
-@click.option(
-    '--project', '-p', 'projects',
-    multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys()
-)
-@click.pass_context
-def wizard(ctx, version, username, password, repository, projects):
-    """
-    Interactive Release Wizard
-    """
-    for project, conf in proj(projects):
-        if subprocess.check_output(
-            ["git", "status", "--porcelain"],
-            cwd=conf["dir"]
-        ):
-            raise RuntimeError("%s repository is not clean." % project)
-
-    # Build test release
-    ctx.invoke(sdist, projects=projects)
-    ctx.invoke(test, projects=projects)
-    click.confirm("Please test the release now. Is it ok?", abort=True)
-
-    # bump version, update docs and contributors
-    ctx.invoke(set_version, version=version, projects=projects)
-    ctx.invoke(docs)
-    ctx.invoke(contributors)
-
-    # version bump commit + tag
-    ctx.invoke(
-        git, args=["commit", "-a", "-m", "bump version"], projects=projects
-    )
-    ctx.invoke(git, args=["tag", "v" + version], projects=projects)
-    ctx.invoke(git, args=["push"], projects=projects)
-    ctx.invoke(git, args=["push", "--tags"], projects=projects)
-
-    # Re-invoke sdist with bumped version
-    ctx.invoke(sdist, projects=projects)
-    click.confirm("All good, can upload to PyPI?", abort=True)
-    ctx.invoke(
-        upload_release,
-        username=username, password=password, repository=repository
-    )
-    click.echo("All done!")
-
-
-if __name__ == "__main__":
-    cli()
diff --git a/rtool.py b/rtool.py
new file mode 100644
index 00000000..c9e697c6
--- /dev/null
+++ b/rtool.py
@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function, division
+
+from os.path import join
+import contextlib
+import os
+import shutil
+import subprocess
+import glob
+import re
+import shlex
+import runpy
+import pprint
+from zipfile import ZipFile
+from tarfile import TarFile
+import platform
+
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+    VENV_BIN = "Scripts"
+else:
+    VENV_BIN = "bin"
+
+if platform.system() == "Windows":
+    def Archive(name):
+        a = ZipFile(name + ".zip","w")
+        a.add = a.write
+        return a
+else:
+    def Archive(name):
+        a = TarFile(name + ".tar.gz", "w:gz")
+        return a
+
+
+RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__)))
+DIST_DIR = join(RELEASE_DIR, "dist")
+ROOT_DIR = join(RELEASE_DIR, "..")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
+
+VENV_DIR = join(BUILD_DIR, "venv")
+VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
+VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
+
+
+ALL_PROJECTS = {
+    "netlib": {
+        "tools": [],
+        "vfile": join(ROOT_DIR, "netlib/netlib/version.py"),
+        "dir": join(ROOT_DIR, "netlib")
+    },
+    "pathod": {
+        "tools": ["pathod", "pathoc"],
+        "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"),
+        "dir": join(ROOT_DIR, "pathod")
+    },
+    "mitmproxy": {
+        "tools": ["mitmproxy", "mitmdump", "mitmweb"],
+        "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"),
+        "dir": join(ROOT_DIR, "mitmproxy")
+    }
+}
+if platform.system() == "Windows":
+    ALL_PROJECTS["mitmproxy"]["tools"].remove("mitmproxy")
+
+projects = {}
+
+def version(project):
+    return runpy.run_path(projects[project]["vfile"])["VERSION"]
+
+def sdist_name(project):
+	return "{project}-{version}.tar.gz".format(project=project, version=version(project))
+
+@contextlib.contextmanager
+def empty_pythonpath():
+    """
+    Make sure that the regular python installation is not on the python path,
+    which would give us access to modules installed outside of our virtualenv.
+    """
+    pythonpath = os.environ.get("PYTHONPATH","")
+    os.environ["PYTHONPATH"] = ""
+    yield
+    os.environ["PYTHONPATH"] = pythonpath
+
+
+@contextlib.contextmanager
+def chdir(path):
+    old_dir = os.getcwd()
+    os.chdir(path)
+    yield
+    os.chdir(old_dir)
+
+
+@click.group(chain=True)
+@click.option(
+    '--project', '-p',
+    multiple=True, type=click.Choice(ALL_PROJECTS.keys()), default=ALL_PROJECTS.keys()
+)
+def cli(project):
+    """
+    mitmproxy build tool
+    """
+    for name in project:
+        projects[name] = ALL_PROJECTS[name]
+
+@cli.command("contributors")
+def contributors():
+    """
+    Update CONTRIBUTORS.md
+    """
+    for project, conf in projects.items():
+        with chdir(conf["dir"]):
+            print("Updating %s/CONTRIBUTORS..." % project)
+            contributors_data = subprocess.check_output(
+                shlex.split("git shortlog -n -s")
+            )
+            with open("CONTRIBUTORS", "w") as f:
+                f.write(contributors_data)
+
+@cli.command("set-version")
+@click.argument('version')
+def set_version(version):
+    """
+    Update version information
+    """
+    print("Update versions...")
+    version = ", ".join(version.split("."))
+    for p, conf in projects.items():
+        print("Update %s..." % os.path.normpath(conf["vfile"]))
+        with open(conf["vfile"], "rb") as f:
+            content = f.read()
+        new_content = re.sub(
+            r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version,
+            content
+        )
+        with open(conf["vfile"], "wb") as f:
+            f.write(new_content)
+
+
+@cli.command("git")
+@click.argument('args', nargs=-1, required=True)
+def git(args):
+    """
+    Run a git command on every project
+    """
+    args = ["git"] + list(args)
+    for project, conf in projects.items():
+        print("%s> %s..." % (project, " ".join(args)))
+        subprocess.check_call(
+            args,
+            cwd=conf["dir"]
+        )
+
+
+@cli.command("sdist")
+def sdist():
+    """
+    Build a source distribution
+    """
+    with empty_pythonpath():
+        print("Building release...")
+        if os.path.exists(DIST_DIR):
+            shutil.rmtree(DIST_DIR)
+        for project, conf in projects.items():
+            print("Creating %s source distribution..." % project)
+            subprocess.check_call(
+                [
+                    "python", "./setup.py",
+                    "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar"
+                ],
+                cwd=conf["dir"]
+            )
+
+        print("Creating virtualenv for test install...")
+        if os.path.exists(VENV_DIR):
+            shutil.rmtree(VENV_DIR)
+        subprocess.check_call(["virtualenv", "-q", VENV_DIR])
+
+        with chdir(DIST_DIR):
+            for project, conf in projects.items():
+                print("Installing %s..." % project)
+                subprocess.check_call([VENV_PIP, "install", "-q", sdist_name(project)])
+
+            print("Running binaries...")
+            for project, conf in projects.items():
+                for tool in conf["tools"]:
+                    tool = join(VENV_DIR, VENV_BIN, tool)
+                    print("> %s --version" % tool)
+                    print(subprocess.check_output([tool, "--version"]))
+
+            print("Virtualenv available for further testing:")
+            print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate")))
+
+
+@cli.command("bdist")
+@click.option('--use-existing-sdist/--no-use-existing-sdist', default=False)
+@click.pass_context
+def bdist(ctx, use_existing_sdist):
+    """
+    Build a binary distribution
+    """
+    if os.path.exists(PYINSTALLER_TEMP):
+        shutil.rmtree(PYINSTALLER_TEMP)
+    if os.path.exists(PYINSTALLER_DIST):
+        shutil.rmtree(PYINSTALLER_DIST)
+
+    if not use_existing_sdist:
+        ctx.invoke(sdist)
+
+    print("Installing PyInstaller...")
+    subprocess.check_call([VENV_PIP, "install", "-q", "PyInstaller~=3.0.0"])
+
+    for p, conf in projects.items():
+        if conf["tools"]:
+            archive_name = "{project}-{version}-{platform}".format(
+                project=p, 
+                version=version(p), 
+                platform=platform.system()
+            )
+            with Archive(join(DIST_DIR, archive_name)) as archive:
+                for tool in conf["tools"]:
+                    spec = join(conf["dir"], "release", "%s.spec" % tool)
+                    print("Building %s binary..." % tool)
+                    subprocess.check_call(
+                        [
+                            VENV_PYINSTALLER,
+                            "--clean",
+                            "--workpath", PYINSTALLER_TEMP,
+                            "--distpath", PYINSTALLER_DIST,
+                            # This is PyInstaller, so setting a 
+                            # different log level obviously breaks it :-)
+                            # "--log-level", "WARN",
+                            spec
+                        ]
+                    )
+
+                    # Test if it works at all O:-)
+                    executable = join(PYINSTALLER_DIST, tool)
+                    if platform.system() == "Windows":
+                        executable += ".exe"
+                    print("Testinng %s..." % executable)
+                    subprocess.check_call([executable, "--version"])
+
+                    archive.add(executable, os.path.basename(executable))
+
+
+@cli.command("upload")
+@click.option('--username', prompt=True)
+@click.password_option(confirmation_prompt=False)
+@click.option('--repository', default="pypi")
+def upload_release(username, password, repository):
+    """
+    Upload source distributions to PyPI
+    """
+    
+    for project in projects.keys():
+	    print("Uploading {} to {}...".format(project, repository))
+	    subprocess.check_call([
+	        "twine",
+	        "upload",
+	        "-u", username,
+	        "-p", password,
+	        "-r", repository,
+	        join(DIST_DIR, sdist_name(project))
+	    ])
+
+
+@cli.command("wizard")
+@click.option('--version', prompt=True)
+@click.option('--username', prompt="PyPI Username")
+@click.password_option(confirmation_prompt=False, prompt="PyPI Password")
+@click.option('--repository', default="pypi")
+@click.pass_context
+def wizard(ctx, version, username, password, repository):
+    """
+    Interactive Release Wizard
+    """
+    for project, conf in projects.items():
+        is_dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=conf["dir"])
+        if is_dirty:
+            raise RuntimeError("%s repository is not clean." % project)
+
+    # Build test release
+    ctx.invoke(bdist)
+    click.confirm("Please test the release now. Is it ok?", abort=True)
+
+    # bump version, update docs and contributors
+    ctx.invoke(set_version, version=version)
+    ctx.invoke(contributors)
+
+    # version bump commit + tag
+    ctx.invoke(
+        git, args=["commit", "-a", "-m", "bump version"]
+    )
+    ctx.invoke(git, args=["tag", version])
+    ctx.invoke(git, args=["push"])
+    ctx.invoke(git, args=["push", "--tags"])
+
+    # Re-invoke sdist with bumped version
+    ctx.invoke(sdist)
+    click.confirm("All good, can upload sdist to PyPI?", abort=True)
+    ctx.invoke(
+        upload_release,
+        username=username, password=password, repository=repository
+    )
+    click.echo("All done!")
+
+
+if __name__ == "__main__":
+    cli()
diff --git a/setup.py b/setup.py
new file mode 100644
index 00000000..06ac230d
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,16 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='mitmproxy-rtool',
+    version='1.0',
+    py_modules=['rtool'],
+    install_requires=[
+        'click~=6.2',
+        'twine~=1.6.4',
+    ],
+    entry_points={
+        'console_scripts': [
+            'rtool=rtool:cli',
+        ],
+    },
+)