update rtool

1 files changed, 315 insertions, 0 deletions

diff --git a/rtool.py b/rtool.py
new file mode 100644
index 00000000..c9e697c6
--- /dev/null
+++ b/rtool.py
@@ -0,0 +1,315 @@
+#!/usr/bin/env python
+from __future__ import absolute_import, print_function, division
+
+from os.path import join
+import contextlib
+import os
+import shutil
+import subprocess
+import glob
+import re
+import shlex
+import runpy
+import pprint
+from zipfile import ZipFile
+from tarfile import TarFile
+import platform
+
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+    VENV_BIN = "Scripts"
+else:
+    VENV_BIN = "bin"
+
+if platform.system() == "Windows":
+    def Archive(name):
+        a = ZipFile(name + ".zip","w")
+        a.add = a.write
+        return a
+else:
+    def Archive(name):
+        a = TarFile(name + ".tar.gz", "w:gz")
+        return a
+
+
+RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__)))
+DIST_DIR = join(RELEASE_DIR, "dist")
+ROOT_DIR = join(RELEASE_DIR, "..")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
+
+VENV_DIR = join(BUILD_DIR, "venv")
+VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
+VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
+
+
+ALL_PROJECTS = {
+    "netlib": {
+        "tools": [],
+        "vfile": join(ROOT_DIR, "netlib/netlib/version.py"),
+        "dir": join(ROOT_DIR, "netlib")
+    },
+    "pathod": {
+        "tools": ["pathod", "pathoc"],
+        "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"),
+        "dir": join(ROOT_DIR, "pathod")
+    },
+    "mitmproxy": {
+        "tools": ["mitmproxy", "mitmdump", "mitmweb"],
+        "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"),
+        "dir": join(ROOT_DIR, "mitmproxy")
+    }
+}
+if platform.system() == "Windows":
+    ALL_PROJECTS["mitmproxy"]["tools"].remove("mitmproxy")
+
+projects = {}
+
+def version(project):
+    return runpy.run_path(projects[project]["vfile"])["VERSION"]
+
+def sdist_name(project):
+	return "{project}-{version}.tar.gz".format(project=project, version=version(project))
+
+@contextlib.contextmanager
+def empty_pythonpath():
+    """
+    Make sure that the regular python installation is not on the python path,
+    which would give us access to modules installed outside of our virtualenv.
+    """
+    pythonpath = os.environ.get("PYTHONPATH","")
+    os.environ["PYTHONPATH"] = ""
+    yield
+    os.environ["PYTHONPATH"] = pythonpath
+
+
+@contextlib.contextmanager
+def chdir(path):
+    old_dir = os.getcwd()
+    os.chdir(path)
+    yield
+    os.chdir(old_dir)
+
+
+@click.group(chain=True)
+@click.option(
+    '--project', '-p',
+    multiple=True, type=click.Choice(ALL_PROJECTS.keys()), default=ALL_PROJECTS.keys()
+)
+def cli(project):
+    """
+    mitmproxy build tool
+    """
+    for name in project:
+        projects[name] = ALL_PROJECTS[name]
+
+@cli.command("contributors")
+def contributors():
+    """
+    Update CONTRIBUTORS.md
+    """
+    for project, conf in projects.items():
+        with chdir(conf["dir"]):
+            print("Updating %s/CONTRIBUTORS..." % project)
+            contributors_data = subprocess.check_output(
+                shlex.split("git shortlog -n -s")
+            )
+            with open("CONTRIBUTORS", "w") as f:
+                f.write(contributors_data)
+
+@cli.command("set-version")
+@click.argument('version')
+def set_version(version):
+    """
+    Update version information
+    """
+    print("Update versions...")
+    version = ", ".join(version.split("."))
+    for p, conf in projects.items():
+        print("Update %s..." % os.path.normpath(conf["vfile"]))
+        with open(conf["vfile"], "rb") as f:
+            content = f.read()
+        new_content = re.sub(
+            r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version,
+            content
+        )
+        with open(conf["vfile"], "wb") as f:
+            f.write(new_content)
+
+
+@cli.command("git")
+@click.argument('args', nargs=-1, required=True)
+def git(args):
+    """
+    Run a git command on every project
+    """
+    args = ["git"] + list(args)
+    for project, conf in projects.items():
+        print("%s> %s..." % (project, " ".join(args)))
+        subprocess.check_call(
+            args,
+            cwd=conf["dir"]
+        )
+
+
+@cli.command("sdist")
+def sdist():
+    """
+    Build a source distribution
+    """
+    with empty_pythonpath():
+        print("Building release...")
+        if os.path.exists(DIST_DIR):
+            shutil.rmtree(DIST_DIR)
+        for project, conf in projects.items():
+            print("Creating %s source distribution..." % project)
+            subprocess.check_call(
+                [
+                    "python", "./setup.py",
+                    "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar"
+                ],
+                cwd=conf["dir"]
+            )
+
+        print("Creating virtualenv for test install...")
+        if os.path.exists(VENV_DIR):
+            shutil.rmtree(VENV_DIR)
+        subprocess.check_call(["virtualenv", "-q", VENV_DIR])
+
+        with chdir(DIST_DIR):
+            for project, conf in projects.items():
+                print("Installing %s..." % project)
+                subprocess.check_call([VENV_PIP, "install", "-q", sdist_name(project)])
+
+            print("Running binaries...")
+            for project, conf in projects.items():
+                for tool in conf["tools"]:
+                    tool = join(VENV_DIR, VENV_BIN, tool)
+                    print("> %s --version" % tool)
+                    print(subprocess.check_output([tool, "--version"]))
+
+            print("Virtualenv available for further testing:")
+            print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate")))
+
+
+@cli.command("bdist")
+@click.option('--use-existing-sdist/--no-use-existing-sdist', default=False)
+@click.pass_context
+def bdist(ctx, use_existing_sdist):
+    """
+    Build a binary distribution
+    """
+    if os.path.exists(PYINSTALLER_TEMP):
+        shutil.rmtree(PYINSTALLER_TEMP)
+    if os.path.exists(PYINSTALLER_DIST):
+        shutil.rmtree(PYINSTALLER_DIST)
+
+    if not use_existing_sdist:
+        ctx.invoke(sdist)
+
+    print("Installing PyInstaller...")
+    subprocess.check_call([VENV_PIP, "install", "-q", "PyInstaller~=3.0.0"])
+
+    for p, conf in projects.items():
+        if conf["tools"]:
+            archive_name = "{project}-{version}-{platform}".format(
+                project=p, 
+                version=version(p), 
+                platform=platform.system()
+            )
+            with Archive(join(DIST_DIR, archive_name)) as archive:
+                for tool in conf["tools"]:
+                    spec = join(conf["dir"], "release", "%s.spec" % tool)
+                    print("Building %s binary..." % tool)
+                    subprocess.check_call(
+                        [
+                            VENV_PYINSTALLER,
+                            "--clean",
+                            "--workpath", PYINSTALLER_TEMP,
+                            "--distpath", PYINSTALLER_DIST,
+                            # This is PyInstaller, so setting a 
+                            # different log level obviously breaks it :-)
+                            # "--log-level", "WARN",
+                            spec
+                        ]
+                    )
+
+                    # Test if it works at all O:-)
+                    executable = join(PYINSTALLER_DIST, tool)
+                    if platform.system() == "Windows":
+                        executable += ".exe"
+                    print("Testinng %s..." % executable)
+                    subprocess.check_call([executable, "--version"])
+
+                    archive.add(executable, os.path.basename(executable))
+
+
+@cli.command("upload")
+@click.option('--username', prompt=True)
+@click.password_option(confirmation_prompt=False)
+@click.option('--repository', default="pypi")
+def upload_release(username, password, repository):
+    """
+    Upload source distributions to PyPI
+    """
+    
+    for project in projects.keys():
+	    print("Uploading {} to {}...".format(project, repository))
+	    subprocess.check_call([
+	        "twine",
+	        "upload",
+	        "-u", username,
+	        "-p", password,
+	        "-r", repository,
+	        join(DIST_DIR, sdist_name(project))
+	    ])
+
+
+@cli.command("wizard")
+@click.option('--version', prompt=True)
+@click.option('--username', prompt="PyPI Username")
+@click.password_option(confirmation_prompt=False, prompt="PyPI Password")
+@click.option('--repository', default="pypi")
+@click.pass_context
+def wizard(ctx, version, username, password, repository):
+    """
+    Interactive Release Wizard
+    """
+    for project, conf in projects.items():
+        is_dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=conf["dir"])
+        if is_dirty:
+            raise RuntimeError("%s repository is not clean." % project)
+
+    # Build test release
+    ctx.invoke(bdist)
+    click.confirm("Please test the release now. Is it ok?", abort=True)
+
+    # bump version, update docs and contributors
+    ctx.invoke(set_version, version=version)
+    ctx.invoke(contributors)
+
+    # version bump commit + tag
+    ctx.invoke(
+        git, args=["commit", "-a", "-m", "bump version"]
+    )
+    ctx.invoke(git, args=["tag", version])
+    ctx.invoke(git, args=["push"])
+    ctx.invoke(git, args=["push", "--tags"])
+
+    # Re-invoke sdist with bumped version
+    ctx.invoke(sdist)
+    click.confirm("All good, can upload sdist to PyPI?", abort=True)
+    ctx.invoke(
+        upload_release,
+        username=username, password=password, repository=repository
+    )
+    click.echo("All done!")
+
+
+if __name__ == "__main__":
+    cli()