diff --git a/doc/release-process.md b/doc/release-process.md index 45dd3af22..d6090dfcd 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -4,130 +4,96 @@ Meta: There should always be a single release engineer to disambiguate responsib ## Pre-release -The following should have been checked well in advance of the release: +### Github Milestone -- All dependencies have been updated as appropriate: - - BDB - - Boost - - ccache - - libgmp - - libsnark (upstream of our fork) - - libsodium - - miniupnpc - - OpenSSL +Ensure all goals for the github milestone are met. If not, remove tickets +or PRs with a comment as to why it is not included. (Running out of time +is a common reason.) +### Pre-release checklist: + +Check that dependencies are properly hosted by looking at the `check-depends` builder: + + https://ci.z.cash/#/builders/1 + +Check that there are no surprising performance regressions: + + https://speed.z.cash + +Ensure that new performance metrics appear on that site. + +### Protocol Safety Checks: + +If this release changes the behavior of the protocol or fixes a serious +bug, verify that a pre-release PR merge updated `PROTOCOL_VERSION` in +`version.h` correctly. + +If this release breaks backwards compatibility or needs to prevent +interaction with software forked projects, change the network magic +numbers. Set the four `pchMessageStart` in `CTestNetParams` in +`chainparams.cpp` to random values. + +Both of these should be done in standard PRs ahead of the release +process. If these were not anticipated correctly, this could block the +release, so if you suspect this is necessary, double check with the +whole engineering team. ## Release process -## A. Define the release version as: +Run the release script, which will verify you are on the latest clean +checkout of master, create a branch, then commit standard automated +changes to that branch locally: - $ ZCASH_RELEASE=MAJOR.MINOR.REVISION(-BUILD_STRING) + $ ./zcutil/make-release.py Example: - $ ZCASH_RELEASE=1.0.0-beta2 + $ ./zcutil/make-release.py v1.0.9 v1.0.8-1 120000 -Also, the following commands use the `ZCASH_RELEASE_PREV` bash variable for the -previous release: +### Create, Review, and Merge the release branch pull request - $ ZCASH_RELEASE_PREV=1.0.0-beta1 +Review the automated changes in git: -## B. Create a new release branch / github PR + $ git log master..HEAD -### B1. Check that you are up-to-date with current master, then create a release branch. +Push the resulting branch to github: -### B2. Update (commit) version and deprecation in sources. + $ git push 'git@github.com:$YOUR_GITHUB_NAME/zcash' $(git rev-parse --abbrev-ref HEAD) -Update the client version in these files: +Then create the PR on github. Complete the standard review process, +then merge, then wait for CI to complete. - README.md - src/clientversion.h - configure.ac - contrib/gitian-descriptors/gitian-linux.yml - -In `configure.ac` and `clientversion.h`: - -- Increment `CLIENT_VERSION_BUILD` according to the following schema: - - - 0-24: `1.0.0-beta1`-`1.0.0-beta25` - - 25-49: `1.0.0-rc1`-`1.0.0-rc25` - - 50: `1.0.0` - - 51-99: `1.0.0-1`-`1.0.0-49` - - (`CLIENT_VERSION_REVISION` rolls over) - - 0-24: `1.0.1-beta1`-`1.0.1-beta25` - -- Change `CLIENT_VERSION_IS_RELEASE` to false while Zcash is in beta-test phase. - -Update `APPROX_RELEASE_HEIGHT` and `WEEKS_UNTIL_DEPRECATION` in `src/deprecation.h` -so that `APPROX_RELEASE_HEIGHT` will be reached shortly after release, and -`WEEKS_UNTIL_DEPRECATION` is the number of weeks from release day until the -deprecation target (as defined by the current deprecation policy). - -If this release changes the behavior of the protocol or fixes a serious bug, we may -also wish to change the `PROTOCOL_VERSION` in `version.h`. - -Commit these changes. (Be sure to do this before building, or else the built binary will include the flag `-dirty`) - -Build by running `./zcutil/build.sh`. - -Then perform the following command: - - $ bash contrib/devtools/gen-manpages.sh - -Commit the changes. - -### B3. Generate release notes - -Run the release-notes.py script to generate release notes and update authors.md file. For example: - - $ python zcutil/release-notes.py --version $ZCASH_RELEASE - -Add the newly created release notes to the Git repository: - - $ git add ./doc/authors.md ./doc/release-notes/release-notes-$ZCASH_RELEASE.md - -Update the Debian package changelog: - - export DEBVERSION=$(echo $ZCASH_RELEASE | sed 's/-beta/~beta/' | sed 's/-rc/~rc/' | sed 's/-/+/') - export DEBEMAIL="${DEBEMAIL:-team@z.cash}" - export DEBFULLNAME="${DEBFULLNAME:-Zcash Company}" - - dch -v $DEBVERSION -D jessie -c contrib/debian/changelog - -(`dch` comes from the devscripts package.) - -### B4. Change the network magics - -If this release breaks backwards compatibility, change the network magic -numbers. Set the four `pchMessageStart` in `CTestNetParams` in `chainparams.cpp` -to random values. - -### B5. Merge the previous changes - -Do the normal pull-request, review, testing process for this release PR. - -## C. Verify code artifact hosting - -### C1. Ensure depends tree is working - -https://ci.z.cash/builders/depends-sources - -### C2. Ensure public parameters work - -Run `./fetch-params.sh`. - -## D. Make tag for the newly merged result +## Make tag for the newly merged result Checkout master and pull the latest version to ensure master is up to date with the release PR which was merged in before. -Check the last commit on the local and remote versions of master to make sure they are the same. + $ git checkout master + $ git pull --ff-only -Then create the git tag: +Check the last commit on the local and remote versions of master to make sure they are the same: - $ git tag -s v${ZCASH_RELEASE} - $ git push origin v${ZCASH_RELEASE} + $ git log -1 -## E. Deploy testnet +The output should include something like, which is created by Homu: + + Auto merge of #4242 - nathan-at-least:release-v1.0.9, r=nathan-at-least + +Then create the git tag. The `-s` means the release tag will be +signed. **CAUTION:** Remember the `v` at the beginning here: + + $ git tag -s v1.0.9 + $ git push origin v1.0.9 + +## Make and deploy deterministic builds + +- Run the [Gitian deterministic build environment](https://github.com/zcash/zcash-gitian) +- Compare the uploaded [build manifests on gitian.sigs](https://github.com/zcash/gitian.sigs) +- If all is well, the DevOps engineer will build the Debian packages and update the + [apt.z.cash package repository](https://apt.z.cash). + +## Post Release Task List + +### Deploy testnet Notify the Zcash DevOps engineer/sysadmin that the release has been tagged. They update some variables in the company's automation code and then run an Ansible playbook, which: @@ -138,26 +104,8 @@ Notify the Zcash DevOps engineer/sysadmin that the release has been tagged. They Then, verify that nodes can connect to the testnet server, and update the guide on the wiki to ensure the correct hostname is listed in the recommended zcash.conf. -## F. Update the 1.0 User Guide +### Update the 1.0 User Guide -## G. Publish the release announcement (blog, zcash-dev, slack) +### Publish the release announcement (blog, zcash-dev, slack) -### G1. Check in with users who opened issues that were resolved in the release - -Contact all users who opened `user support` issues that were resolved in the release, and ask them if the release fixes or improves their issue. - -## H. Make and deploy deterministic builds - -- Run the [Gitian deterministic build environment](https://github.com/zcash/zcash-gitian) -- Compare the uploaded [build manifests on gitian.sigs](https://github.com/zcash/gitian.sigs) -- If all is well, the DevOps engineer will build the Debian packages and update the - [apt.z.cash package repository](https://apt.z.cash). - -## I. Celebrate - -## missing steps -Zcash still needs: - -* thorough pre-release testing (presumably more thorough than standard PR tests) - -* automated release deployment (e.g.: updating build-depends mirror, deploying testnet, etc...) +## Celebrate diff --git a/zcutil/make-release.py b/zcutil/make-release.py new file mode 100755 index 000000000..514d5883f --- /dev/null +++ b/zcutil/make-release.py @@ -0,0 +1,543 @@ +#! /usr/bin/env python2 + +import os +import re +import sys +import logging +import argparse +import subprocess +import traceback +import unittest +import random +from cStringIO import StringIO +from functools import wraps + + +def main(args=sys.argv[1:]): + """ + Perform the final Zcash release process up to the git tag. + """ + opts = parse_args(args) + chdir_to_repo(opts.REPO) + initialize_logging() + logging.debug('argv %r', sys.argv) + + try: + main_logged( + opts.RELEASE_VERSION, + opts.RELEASE_PREV, + opts.RELEASE_HEIGHT, + ) + except SystemExit as e: + logging.error(str(e)) + raise SystemExit(1) + except: + logging.error(traceback.format_exc()) + raise SystemExit(2) + + +def parse_args(args): + p = argparse.ArgumentParser(description=main.__doc__) + p.add_argument( + '--repo', + dest='REPO', + type=str, + help='Path to repository root.', + ) + p.add_argument( + 'RELEASE_VERSION', + type=Version.parse_arg, + help='The release version: vX.Y.Z-N', + ) + p.add_argument( + 'RELEASE_PREV', + type=Version.parse_arg, + help='The previously released version.', + ) + p.add_argument( + 'RELEASE_HEIGHT', + type=int, + help='A block height approximately occuring on release day.', + ) + return p.parse_args(args) + + +# Top-level flow: +def main_logged(release, releaseprev, releaseheight): + verify_releaseprev_tag(releaseprev) + initialize_git(release) + patch_version_in_files(release, releaseprev) + patch_release_height(releaseheight) + commit('Versioning changes for {}.'.format(release.novtext)) + + build() + gen_manpages() + commit('Updated manpages for {}.'.format(release.novtext)) + + gen_release_notes(release) + update_debian_changelog(release) + commit( + 'Updated release notes and changelog for {}.'.format( + release.novtext, + ), + ) + + +def phase(message): + def deco(f): + @wraps(f) + def g(*a, **kw): + logging.info('%s', message) + return f(*a, **kw) + return g + return deco + + +@phase('Checking RELEASE_PREV tag.') +def verify_releaseprev_tag(releaseprev): + candidates = [] + + # Any tag beginning with a 'v' followed by [1-9] must be a version + # matching our Version parser. Tags beginning with v0 may exist from + # upstream and those do not follow our schema and are silently + # ignored. Any other tag is silently ignored. + candidatergx = re.compile('^v[1-9].*$') + + for tag in sh_out('git', 'tag', '--list').splitlines(): + if candidatergx.match(tag): + candidates.append(Version.parse_arg(tag)) + + candidates.sort() + try: + latest = candidates[-1] + except IndexError: + raise SystemExit('No previous releases found by `git tag --list`.') + + if releaseprev != latest: + raise SystemExit( + 'The latest candidate in `git tag --list` is {} not {}' + .format( + latest.vtext, + releaseprev.vtext, + ), + ) + + +@phase('Initializing git.') +def initialize_git(release): + junk = sh_out('git', 'status', '--porcelain') + if junk.strip(): + raise SystemExit('There are uncommitted changes:\n' + junk) + + branch = sh_out('git', 'rev-parse', '--abbrev-ref', 'HEAD').strip() + if branch != 'master': + raise SystemExit( + "Expected branch 'master', found branch {!r}".format( + branch, + ), + ) + + logging.info('Pulling to latest master.') + sh_log('git', 'pull', '--ff-only') + + branch = 'release-' + release.vtext + logging.info('Creating release branch: %r', branch) + sh_log('git', 'checkout', '-b', branch) + return branch + + +@phase('Patching versioning in files.') +def patch_version_in_files(release, releaseprev): + patch_README(release, releaseprev) + patch_clientversion_h(release) + patch_configure_ac(release) + patch_gitian_linux_yml(release, releaseprev) + + +@phase('Patching release height for auto-senescence.') +def patch_release_height(releaseheight): + rgx = re.compile( + r'^(static const int APPROX_RELEASE_HEIGHT = )\d+(;)$', + ) + with PathPatcher('src/deprecation.h') as (inf, outf): + for line in inf: + m = rgx.match(line) + if m is None: + outf.write(line) + else: + [prefix, suffix] = m.groups() + outf.write( + '{}{}{}\n'.format( + prefix, + releaseheight, + suffix, + ), + ) + + +@phase('Building...') +def build(): + nproc = sh_out('nproc').strip() + sh_log('./zcutil/build.sh', '-j', nproc) + + +@phase('Generating manpages.') +def gen_manpages(): + sh_log('./contrib/devtools/gen-manpages.sh') + + +@phase('Generating release notes.') +def gen_release_notes(release): + sh_log('python', './zcutil/release-notes.py', '--version', release.novtext) + sh_log( + 'git', + 'add', + './doc/authors.md', + './doc/release-notes/release-notes-{}.md'.format(release.novtext), + ) + + +@phase('Updating debian changelog.') +def update_debian_changelog(release): + os.environ['DEBEMAIL'] = 'team@z.cash' + os.environ['DEBFULLNAME'] = 'Zcash Company' + sh_log( + 'debchange', + '--newversion', release.debversion, + '--distribution', 'stable', + '--changelog', './contrib/debian/changelog', + '{} release.'.format(release.novtext), + ) + + +# Helper code: +def commit(message): + logging.info('Committing: %r', message) + fullmsg = 'make-release.py: {}'.format(message) + sh_log('git', 'commit', '--all', '-m', fullmsg) + + +def chdir_to_repo(repo): + if repo is None: + dn = os.path.dirname + repo = dn(dn(os.path.abspath(sys.argv[0]))) + os.chdir(repo) + + +def patch_README(release, releaseprev): + with PathPatcher('README.md') as (inf, outf): + firstline = inf.readline() + assert firstline == 'Zcash {}\n'.format(releaseprev.novtext), \ + repr(firstline) + + outf.write('Zcash {}\n'.format(release.novtext)) + outf.write(inf.read()) + + +def patch_clientversion_h(release): + _patch_build_defs( + release, + 'src/clientversion.h', + (r'^(#define CLIENT_VERSION_(MAJOR|MINOR|REVISION|BUILD|IS_RELEASE))' + r' \d+()$'), + ) + + +def patch_configure_ac(release): + _patch_build_defs( + release, + 'configure.ac', + (r'^(define\(_CLIENT_VERSION_(MAJOR|MINOR|REVISION|BUILD|IS_RELEASE),)' + r' \d+(\))$'), + ) + + +def patch_gitian_linux_yml(release, releaseprev): + path = 'contrib/gitian-descriptors/gitian-linux.yml' + with PathPatcher(path) as (inf, outf): + outf.write(inf.readline()) + + secondline = inf.readline() + assert secondline == 'name: "zcash-{}"\n'.format( + releaseprev.novtext + ), repr(secondline) + + outf.write('name: "zcash-{}"\n'.format(release.novtext)) + outf.write(inf.read()) + + +def _patch_build_defs(release, path, pattern): + rgx = re.compile(pattern) + with PathPatcher(path) as (inf, outf): + for line in inf: + m = rgx.match(line) + if m: + prefix, label, suffix = m.groups() + repl = { + 'MAJOR': release.major, + 'MINOR': release.minor, + 'REVISION': release.patch, + 'BUILD': release.build, + 'IS_RELEASE': ( + 'false' if release.build < 50 else 'true' + ), + }[label] + outf.write('{} {}{}\n'.format(prefix, repl, suffix)) + else: + outf.write(line) + + +def initialize_logging(): + logname = './zcash-make-release.log' + fmtr = logging.Formatter( + '%(asctime)s L%(lineno)-4d %(levelname)-5s | %(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + hout = logging.StreamHandler(sys.stdout) + hout.setLevel(logging.INFO) + hout.setFormatter(fmtr) + + hpath = logging.FileHandler(logname, mode='a') + hpath.setLevel(logging.DEBUG) + hpath.setFormatter(fmtr) + + root = logging.getLogger() + root.setLevel(logging.DEBUG) + root.addHandler(hout) + root.addHandler(hpath) + logging.info('zcash make-release.py debug log: %r', logname) + + +def sh_out(*args): + logging.debug('Run (out): %r', args) + return subprocess.check_output(args) + + +def sh_log(*args): + PIPE = subprocess.PIPE + try: + p = subprocess.Popen(args, stdout=PIPE, stderr=PIPE, stdin=None) + except OSError: + logging.error('Error launching %r...', args) + raise + + logging.debug('Run (log PID %r): %r', p.pid, args) + for line in p.stdout: + logging.debug('> %s', line.rstrip()) + status = p.wait() + if status != 0: + raise SystemExit('Nonzero exit status: {!r}'.format(status)) + + +class Version (object): + '''A release version.''' + + RGX = re.compile( + r'^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(beta|rc)?([1-9]\d*))?$', + ) + + @staticmethod + def parse_arg(text): + m = Version.RGX.match(text) + if m is None: + raise argparse.ArgumentTypeError( + 'Could not parse version {!r} against regex {}'.format( + text, + Version.RGX.pattern, + ), + ) + else: + [major, minor, patch, _, betarc, hotfix] = m.groups() + return Version( + int(major), + int(minor), + int(patch), + betarc, + int(hotfix) if hotfix is not None else None, + ) + + def __init__(self, major, minor, patch, betarc, hotfix): + for i in [major, minor, patch]: + assert type(i) is int, i + assert betarc in {None, 'rc', 'beta'}, betarc + assert hotfix is None or type(hotfix) is int, hotfix + if betarc is not None: + assert hotfix is not None, (betarc, hotfix) + + self.major = major + self.minor = minor + self.patch = patch + self.betarc = betarc + self.hotfix = hotfix + + if hotfix is None: + self.build = 50 + else: + assert hotfix > 0, hotfix + if betarc is None: + assert hotfix < 50, hotfix + self.build = 50 + hotfix + else: + assert hotfix < 26, hotfix + self.build = {'beta': 0, 'rc': 25}[betarc] + hotfix - 1 + + @property + def novtext(self): + return self._novtext(debian=False) + + @property + def vtext(self): + return 'v' + self.novtext + + @property + def debversion(self): + return self._novtext(debian=True) + + def _novtext(self, debian): + novtext = '{}.{}.{}'.format(self.major, self.minor, self.patch) + + if self.hotfix is None: + return novtext + else: + assert self.hotfix > 0, self.hotfix + if self.betarc is None: + assert self.hotfix < 50, self.hotfix + sep = '+' if debian else '-' + return '{}{}{}'.format(novtext, sep, self.hotfix) + else: + assert self.hotfix < 26, self.hotfix + sep = '~' if debian else '-' + return '{}{}{}{}'.format( + novtext, + sep, + self.betarc, + self.hotfix, + ) + + def __repr__(self): + return ''.format(self.vtext) + + def _sort_tup(self): + if self.hotfix is None: + prio = 2 + else: + prio = {'beta': 0, 'rc': 1, None: 3}[self.betarc] + + return ( + self.major, + self.minor, + self.patch, + prio, + self.hotfix, + ) + + def __cmp__(self, other): + return cmp(self._sort_tup(), other._sort_tup()) + + +class PathPatcher (object): + def __init__(self, path): + self._path = path + + def __enter__(self): + logging.debug('Patching %r', self._path) + self._inf = file(self._path, 'r') + self._outf = StringIO() + return (self._inf, self._outf) + + def __exit__(self, et, ev, tb): + if (et, ev, tb) == (None, None, None): + self._inf.close() + with file(self._path, 'w') as f: + f.write(self._outf.getvalue()) + + +# Unit Tests +class TestVersion (unittest.TestCase): + ValidVersionsAndBuilds = [ + # These are taken from: git tag --list | grep '^v1' + ('v1.0.0-beta1', 0), + ('v1.0.0-beta2', 1), + ('v1.0.0-rc1', 25), + ('v1.0.0-rc2', 26), + ('v1.0.0-rc3', 27), + ('v1.0.0-rc4', 28), + ('v1.0.0', 50), + ('v1.0.1', 50), + ('v1.0.2', 50), + ('v1.0.3', 50), + ('v1.0.4', 50), + ('v1.0.5', 50), + ('v1.0.6', 50), + ('v1.0.7-1', 51), + ('v1.0.8', 50), + ('v1.0.8-1', 51), + ('v1.0.9', 50), + ('v1.0.10', 50), + ('v7.42.1000', 50), + ] + + ValidVersions = [ + v + for (v, _) + in ValidVersionsAndBuilds + ] + + def test_arg_parse_and_vtext_identity(self): + for case in self.ValidVersions: + v = Version.parse_arg(case) + self.assertEqual(v.vtext, case) + + def test_arg_parse_negatives(self): + cases = [ + 'v07.0.0', + 'v1.0.03', + 'v1.2.3-0', # Hotfix numbers must begin w/ 1 + 'v1.2.3~0', + 'v1.2.3+0', + '1.2.3', + ] + + for case in cases: + self.assertRaises( + argparse.ArgumentTypeError, + Version.parse_arg, + case, + ) + + def test_version_sort(self): + expected = [Version.parse_arg(v) for v in self.ValidVersions] + + rng = random.Random() + rng.seed(0) + + for _ in range(1024): + vec = list(expected) + rng.shuffle(vec) + vec.sort() + self.assertEqual(vec, expected) + + def test_build_nums(self): + for (text, expected) in self.ValidVersionsAndBuilds: + version = Version.parse_arg(text) + self.assertEqual(version.build, expected) + + +if __name__ == '__main__': + if len(sys.argv) == 2 and sys.argv[1] == '--help': + main() + else: + actualargs = sys.argv + sys.argv = [sys.argv[0], '--verbose'] + + print '=== Self Test ===' + try: + unittest.main() + except SystemExit as e: + if e.args[0] != 0: + raise + + sys.argv = actualargs + print '=== Running ===' + main() diff --git a/zcutil/release-notes.py b/zcutil/release-notes.py old mode 100644 new mode 100755