diff options
Diffstat (limited to 'release')
| -rwxr-xr-x | release/build.py | 241 | ||||
| -rwxr-xr-x | release/contributors | 2 | ||||
| -rw-r--r-- | release/release-checklist | 37 | ||||
| -rw-r--r-- | release/release-checklist.md | 55 |
4 files changed, 296 insertions, 39 deletions
diff --git a/release/build.py b/release/build.py new file mode 100755 index 00000000..f45efc1c --- /dev/null +++ b/release/build.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python + +from __future__ import (absolute_import, print_function, division, unicode_literals) +from contextlib import contextmanager +from os.path import dirname, realpath, join, exists, normpath +import os +import shutil +import subprocess +import glob +import re +from shlex import split +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if os.name == "nt": + venv_bin = "Scripts" +else: + venv_bin = "bin" + +root_dir = join(dirname(realpath(__file__)), "..", "..") +mitmproxy_dir = join(root_dir, "mitmproxy") +dist_dir = join(mitmproxy_dir, "dist") +test_venv_dir = join(root_dir, "venv.mitmproxy-release") + +all_projects = ("netlib", "pathod", "mitmproxy") +tools = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathod", "pathoc"], + "netlib": [] +} +if os.name == "nt": + tools["mitmproxy"].remove("mitmproxy") +version_files = { + "mitmproxy": normpath(join(root_dir, "mitmproxy/libmproxy/version.py")), + "pathod": normpath(join(root_dir, "pathod/libpathod/version.py")), + "netlib": normpath(join(root_dir, "netlib/netlib/version.py")), +} + + +@contextmanager +def empty_pythonpath(): + """ + Make sure that the regular python installation is not on the python path, + which would give us access to modules installed outside of our virtualenv. + """ + pythonpath = os.environ["PYTHONPATH"] + os.environ["PYTHONPATH"] = "" + yield + os.environ["PYTHONPATH"] = pythonpath + + +@contextmanager +def chdir(path): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("contributors") +def contributors(): + """ + Update CONTRIBUTORS.md + """ + print("Updating CONTRIBUTORS.md...") + contributors_data = subprocess.check_output(split("git shortlog -n -s")) + with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f: + f.write(contributors_data) + + +@cli.command("docs") +def docs(): + """ + Render the docs + """ + print("Rendering the docs...") + subprocess.check_call([ + "cshape", + join(mitmproxy_dir, "doc-src"), + join(mitmproxy_dir, "doc") + ]) + + +@cli.command("set-version") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.argument('version') +def set_version(projects, version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for project, version_file in version_files.items(): + if project not in projects: + continue + print("Update %s..." % version_file) + with open(version_file, "rb") as f: + content = f.read() + new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content) + with open(version_file, "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.argument('args', nargs=-1, required=True) +def git(projects, args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project in projects: + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=join(root_dir, project) + ) + + +@cli.command("sdist") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +def sdist(projects): + """ + Build a source distribution + """ + with empty_pythonpath(): + print("Building release...") + if exists(dist_dir): + shutil.rmtree(dist_dir) + for project in projects: + print("Creating %s source distribution..." % project) + subprocess.check_call( + ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"], + cwd=join(root_dir, project) + ) + + +@cli.command("test") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.pass_context +def test(ctx, projects): + """ + Test the source distribution + """ + if not exists(dist_dir): + ctx.invoke(sdist) + + with empty_pythonpath(): + print("Creating virtualenv for test install...") + if exists(test_venv_dir): + shutil.rmtree(test_venv_dir) + subprocess.check_call(["virtualenv", "-q", test_venv_dir]) + + pip = join(test_venv_dir, venv_bin, "pip") + with chdir(dist_dir): + for project in projects: + print("Installing %s..." % project) + dist = glob.glob("./%s*" % project)[0] + subprocess.check_call([pip, "install", "-q", dist]) + + print("Running binaries...") + for project in projects: + for tool in tools[project]: + tool = join(test_venv_dir, venv_bin, tool) + print(tool) + print(subprocess.check_output([tool, "--version"])) + + print("Virtualenv available for further testing:") + print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate"))) + + +@cli.command("upload") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ + print("Uploading distributions...") + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + "%s/*" % dist_dir + ]) + + +# TODO: Fully automate build process. +# This wizard is missing OSX builds and updating mitmproxy.org. +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt="PyPI Username") +@click.password_option(confirmation_prompt=False, prompt="PyPI Password") +@click.option('--repository', default="pypi") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.pass_context +def wizard(ctx, version, username, password, repository, projects): + """ + Interactive Release Wizard + """ + for project in projects: + if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)): + raise RuntimeError("%s repository is not clean." % project) + + # Build test release + ctx.invoke(sdist, projects=projects) + ctx.invoke(test, projects=projects) + click.confirm("Please test the release now. Is it ok?", abort=True) + + # bump version, update docs and contributors + ctx.invoke(set_version, version=version, projects=projects) + ctx.invoke(docs) + ctx.invoke(contributors) + + # version bump commit + tag + ctx.invoke(git, args=["commit", "-a", "-m", "bump version"], projects=projects) + ctx.invoke(git, args=["tag", "v" + version], projects=projects) + ctx.invoke(git, args=["push"], projects=projects) + ctx.invoke(git, args=["push", "--tags"], projects=projects) + + # Re-invoke sdist with bumped version + ctx.invoke(sdist, projects=projects) + click.confirm("All good, can upload to PyPI?", abort=True) + ctx.invoke(upload_release, username=username, password=password, repository=repository) + click.echo("All done!") + + +if __name__ == "__main__": + cli() diff --git a/release/contributors b/release/contributors deleted file mode 100755 index a7518219..00000000 --- a/release/contributors +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -git shortlog -n -s diff --git a/release/release-checklist b/release/release-checklist deleted file mode 100644 index 84a7152e..00000000 --- a/release/release-checklist +++ /dev/null @@ -1,37 +0,0 @@ - -- Check the version number: - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py - -- Ensure that the website style assets have been compiled for production, and -synced to the docs. - -- Render the docs: - cshape doc-src doc - -- Run the test release, make sure the output is sensible - ./release/test-release - -- Build the OSX binaries - - Follow instructions in osxbinaries - - Move to download dir: - mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download - -- Build the sources for each project: - python ./setup.py sdist - mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download - -- Tag with the version number, and do: - git push --tags - -- Upload to pypi for each project: - - python ./setup.py sdist upload - -- Now bump the version number to be ready for the next cycle: - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py diff --git a/release/release-checklist.md b/release/release-checklist.md new file mode 100644 index 00000000..e6d9ae1f --- /dev/null +++ b/release/release-checklist.md @@ -0,0 +1,55 @@ +# Release Checklist + +## Test + + - Create the source distributions, make sure the output is sensible: + `./release/build.py release` + All source distributions can be found in `./dist`. + + - Test the source distributions: + `./release/build.py test` + This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. + +## Release + + - Verify that repositories are in a clean state: + `./release/build.py git status` + + - Update the version number in `version.py` for all projects: + `./release/build.py set-version 0.13` + + - Ensure that the website style assets have been compiled for production, and synced to the docs. + + - Render the docs, update CONTRIBUTORS file: + `./release/build.py docs contributors` + + - Make version bump commit for all projects, tag and push it: + `./release/build.py git commit -am "bump version"` + `./release/build.py git tag v0.13` + `./release/build.py git push --tags` + + - Recreate the source distributions with updated version information: + `./release/build.py sdist` + + - Build the OSX binaries + - Follow instructions in osx-binaries + - Move to download dir: + `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` + + - Move all source distributions from `./dist` to the server: + `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` + + - Upload distributions in `./dist` to PyPI: + `./release/build.py upload` + You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. + ([more info](https://tom-christie.github.io/articles/pypi/)) + + - Now bump the version number to be ready for the next cycle: + + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + We should probably just leave it as-is and only bump once we actually do the next release. + + Also, we need a release policy. I propose the following: + - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. + - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)`
\ No newline at end of file |