Merge pull request #3131 from cortesi/postrelease

Post-release updates

3 files changed, 105 insertions, 190 deletions

diff --git a/release/README.md b/release/README.md
index ab9992cc..08972076 100644
--- a/release/README.md
+++ b/release/README.md
@@ -1,26 +1,40 @@
 # Release Checklist
 
-Make sure run all these steps on the correct branch you want to create a new release for!
-- Verify `mitmproxy/version.py`
+Make sure to run all these steps on the correct branch you want to create a new
+release for! The command examples assume that you have a git remote called
+`upstream` that points to the `mitmproxy/mitmproxy` repo.
+
+- Verify that `mitmproxy/version.py` is correct
 - Update CHANGELOG
 - Verify that all CI tests pass
+- Create a major version branch - e.g. `v4.x`. Assuming you have a remote repo called `upstream` that points to the mitmproxy/mitmproxy repo::
+  - `git checkout -b v4.x upstream/master`
+  - `git push -u upstream v4.x`
 - Tag the release and push to Github
-  - For alphas, betas, and release candidates, use lightweight tags.
-    This is necessary so that the .devXXXX counter does not reset.
-  - For final releases, use annotated tags.
-    This makes the .devXXXX counter reset.
+  - For alphas, betas, and release candidates, use lightweight tags. This is
+    necessary so that the .devXXXX counter does not reset.
+  - For final releases, use annotated tags. This makes the .devXXXX counter reset.
+    - `git tag -a v4.0.0 -m v4.0.0`
+    - `git push upstream v4.0.0`
 - Wait for tag CI to complete
 
 ## GitHub Release
-- Create release notice on Github [here](https://github.com/mitmproxy/mitmproxy/releases/new)
-- Attach all files from the new release folder on https://snapshots.mitmproxy.org
+- Create release notice on Github
+  [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already
+  auto-created by the tag.
+- We DO NOT upload release artifacts to GitHub anymore. Simply add the
+  following snippet to the notice:
+  `You can find the latest release packages on our snapshot server: https://snapshots.mitmproxy.org/v<version number here>`
 
 ## PyPi
-- `tox -e rtool -- upload-release`
+- The created wheel is uploaded to PyPi automatically
+- Please check https://pypi.python.org/pypi/mitmproxy about the latest version
 
 ## Homebrew
-- `tox -e rtool -- homebrew-pr`
-- The Homebrew maintainers are typically very fast and detect our new relese within a day, but we can be a nice citizen and create the PR ourself.
+- The Homebrew maintainers are typically very fast and detect our new relese
+  within a day.
+- If you feel the need, you can run this from a macOS machine:
+  `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>`
 
 ## Docker
 - Update docker-releases repo
@@ -29,8 +43,8 @@ Make sure run all these steps on the correct branch you want to create a new rel
     * Creating a fresh venv, pip-installing the new wheel in there, and then export all packages:
     * `virtualenv -ppython3.6 venv && source venv/bin/activate && pip install mitmproxy && pip freeze`
   - Tag the commit with the correct version
-    * `2.0.0` for new major versions
-    * `2.0.2` for new patch versions
+    * `v2.0.0` for new major versions
+    * `v2.0.2` for new patch versions
 - Update `latest` tag [here](https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/)
 - Check that the build for this tag succeeds [https://hub.docker.com/r/mitmproxy/mitmproxy/builds/](here)
 - If build failed:
@@ -41,10 +55,17 @@ Make sure run all these steps on the correct branch you want to create a new rel
   - Check the build details page again
 
 ## Website
- - Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml
- - `./build && ./upload-test`
- - If everything looks alright: `./upload-prod`
+ - Update version here:
+   https://github.com/mitmproxy/www/blob/master/src/config.toml
+ - Run `./build && ./upload-test`
+ - If everything looks alright, run `./upload-prod`
 
-## Prepare for next release
+## Docs
+  - Make sure you've uploaded the previous version's docs to archive
+  - If everything looks alright:
+    - `./build-current`
+    - `./upload-stable`
 
- - Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
+## Prepare for next release
+ - Last but not least, bump the version on master in
+   [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
diff --git a/release/ci.py b/release/ci.py
index 94b1f13d..a5000879 100755
--- a/release/ci.py
+++ b/release/ci.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import glob
+import re
 import contextlib
 import os
 import platform
@@ -10,7 +12,6 @@ import tarfile
 import zipfile
 from os.path import join, abspath, dirname, exists, basename
 
-import cryptography.fernet
 import click
 
 # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
@@ -73,17 +74,17 @@ TOOLS = [
 TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
 BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
 if TAG:
-    VERSION = TAG
+    VERSION = re.sub('^v', '', TAG)
     UPLOAD_DIR = VERSION
 elif BRANCH:
-    VERSION = BRANCH
+    VERSION = re.sub('^v', '', BRANCH)
     UPLOAD_DIR = "branches/%s" % VERSION
 else:
     print("Could not establish build name - exiting." % BRANCH)
     sys.exit(0)
 
-
 print("BUILD VERSION=%s" % VERSION)
+print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
 
 
 def archive_name(bdist: str) -> str:
@@ -99,23 +100,6 @@ def archive_name(bdist: str) -> str:
     )
 
 
-def wheel_name() -> str:
-    return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
-
-
-def installer_name() -> str:
-    ext = {
-        "Windows": "exe",
-        "Darwin": "dmg",
-        "Linux": "run"
-    }[platform.system()]
-    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
-        version=VERSION,
-        platform=PLATFORM_TAG,
-        ext=ext,
-    )
-
-
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -134,7 +118,7 @@ def cli():
 
 @cli.command("info")
 def info():
-    print("Version: %s" % VERSION)
+    click.echo("Version: %s" % VERSION)
 
 
 @cli.command("build")
@@ -142,23 +126,41 @@ def build():
     """
     Build a binary distribution
     """
+    os.makedirs(DIST_DIR, exist_ok=True)
+    if "WHEEL" in os.environ:
+        build_wheel()
+    else:
+        click.echo("Not building wheels.")
+    build_pyinstaller()
+
+
+def build_wheel():
+    click.echo("Building wheel...")
+    subprocess.check_call([
+        "python",
+        "setup.py",
+        "-q",
+        "bdist_wheel",
+        "--dist-dir", DIST_DIR,
+    ])
+
+    whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+    click.echo("Found wheel package: {}".format(whl))
+
+    subprocess.check_call([
+        "tox",
+        "-e", "wheeltest",
+        "--",
+        whl
+    ])
+
+
+def build_pyinstaller():
     if exists(PYINSTALLER_TEMP):
         shutil.rmtree(PYINSTALLER_TEMP)
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    if "WHEEL" in os.environ:
-        print("Building wheel...")
-        subprocess.check_call(
-            [
-                "python",
-                "setup.py", "-q", "bdist_wheel",
-                "--dist-dir", "release/dist",
-            ]
-        )
-
     for bdist, tools in sorted(BDISTS.items()):
         with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
             for tool in tools:
@@ -168,7 +170,7 @@ def build():
                 # This is PyInstaller, so it messes up paths.
                 # We need to make sure that we are in the spec folder.
                 with chdir(PYINSTALLER_SPEC):
-                    print("Building %s binary..." % tool)
+                    click.echo("Building %s binary..." % tool)
                     excludes = []
                     if tool != "mitmweb":
                         excludes.append("mitmproxy.tools.web")
@@ -209,11 +211,11 @@ def build():
                     )
                     executable = executable.replace("_main", "")
 
-                print("> %s --version" % executable)
-                print(subprocess.check_output([executable, "--version"]).decode())
+                click.echo("> %s --version" % executable)
+                click.echo(subprocess.check_output([executable, "--version"]).decode())
 
                 archive.add(executable, basename(executable))
-        print("Packed {}.".format(archive_name(bdist)))
+        click.echo("Packed {}.".format(archive_name(bdist)))
 
 
 def is_pr():
@@ -229,34 +231,40 @@ def is_pr():
 @cli.command("upload")
 def upload():
     """
-        Upload snapshot to snapshot server
+        Upload build artifacts to snapshot server and
+        upload wheel package to PyPi
     """
     # This requires some explanation. The AWS access keys are only exposed to
     # privileged builds - that is, they are not available to PRs from forks.
     # However, they ARE exposed to PRs from a branch within the main repo. This
     # check catches that corner case, and prevents an inadvertent upload.
     if is_pr():
-        print("Refusing to upload a pull request")
+        click.echo("Refusing to upload a pull request")
         return
+
     if "AWS_ACCESS_KEY_ID" in os.environ:
-        subprocess.check_call(
-            [
-                "aws", "s3", "cp",
-                "--acl", "public-read",
-                DIST_DIR + "/",
-                "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
-                "--recursive",
-            ]
-        )
-
-
-@cli.command("decrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def decrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.decrypt(infile.read()))
+        subprocess.check_call([
+            "aws", "s3", "cp",
+            "--acl", "public-read",
+            DIST_DIR + "/",
+            "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+            "--recursive",
+        ])
+
+    upload_pypi = (
+        TAG and
+        "WHEEL" in os.environ and
+        "TWINE_USERNAME" in os.environ and
+        "TWINE_PASSWORD" in os.environ
+    )
+    if upload_pypi:
+        filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+        click.echo("Uploading {} to PyPi...".format(filename))
+        subprocess.check_call([
+            "twine",
+            "upload",
+            join(DIST_DIR, filename)
+        ])
 
 
 if __name__ == "__main__":
diff --git a/release/rtool.py b/release/rtool.py
deleted file mode 100755
index 0009cf89..00000000
--- a/release/rtool.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python3
-
-import contextlib
-import os
-import sys
-import platform
-import runpy
-import shlex
-import subprocess
-from os.path import join, abspath, dirname
-
-import cryptography.fernet
-import click
-
-
-ROOT_DIR = abspath(join(dirname(__file__), ".."))
-RELEASE_DIR = join(ROOT_DIR, "release")
-DIST_DIR = join(RELEASE_DIR, "dist")
-VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-
-
-def git(args: str) -> str:
-    with chdir(ROOT_DIR):
-        return subprocess.check_output(["git"] + shlex.split(args)).decode()
-
-
-def get_version(dev: bool = False, build: bool = False) -> str:
-    x = runpy.run_path(VERSION_FILE)
-    return x["get_version"](dev, build, True)
-
-
-def wheel_name() -> str:
-    return "mitmproxy-{version}-py3-none-any.whl".format(
-        version=get_version(True),
-    )
-
-
-@contextlib.contextmanager
-def chdir(path: str):
-    old_dir = os.getcwd()
-    os.chdir(path)
-    yield
-    os.chdir(old_dir)
-
-
-@click.group(chain=True)
-def cli():
-    """
-    mitmproxy build tool
-    """
-    pass
-
-
-@cli.command("contributors")
-def contributors():
-    """
-    Update CONTRIBUTORS.md
-    """
-    with chdir(ROOT_DIR):
-        print("Updating CONTRIBUTORS...")
-        contributors_data = git("shortlog -n -s")
-        with open("CONTRIBUTORS", "wb") as f:
-            f.write(contributors_data.encode())
-
-
-@cli.command("upload-release")
-@click.option('--username', prompt=True)
-@click.password_option(confirmation_prompt=False)
-@click.option('--repository', default="pypi")
-def upload_release(username, password, repository):
-    """
-    Upload wheels to PyPI
-    """
-    filename = wheel_name()
-    print("Uploading {} to {}...".format(filename, repository))
-    subprocess.check_call([
-        "twine",
-        "upload",
-        "-u", username,
-        "-p", password,
-        "-r", repository,
-        join(DIST_DIR, filename)
-    ])
-
-
-@cli.command("homebrew-pr")
-def homebrew_pr():
-    """
-    Create a new Homebrew PR
-    """
-    if platform.system() != "Darwin":
-        print("You need to run this on macOS to create a new Homebrew PR. Sorry.")
-        sys.exit(1)
-
-    print("Creating a new PR with Homebrew...")
-    subprocess.check_call([
-        "brew",
-        "bump-formula-pr",
-        "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}".format(get_version()),
-        "mitmproxy",
-    ])
-
-
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.encrypt(infile.read()))
-
-
-if __name__ == "__main__":
-    cli()