From aadf3aa15961fac06ca95f50f8034a20535ed002 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Fri, 6 Oct 2017 00:21:05 +0100
Subject: [PATCH 1/5] Replace full-test-suite.sh with a new test suite driver
 script

This will be the canonical location for the entire Zcash merge test suite.
---
 qa/zcash/full-test-suite.sh | 47 --------------------
 qa/zcash/full_test_suite.py | 87 +++++++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+), 47 deletions(-)
 delete mode 100755 qa/zcash/full-test-suite.sh
 create mode 100755 qa/zcash/full_test_suite.py

diff --git a/qa/zcash/full-test-suite.sh b/qa/zcash/full-test-suite.sh
deleted file mode 100755
index 7860b105a..000000000
--- a/qa/zcash/full-test-suite.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-#
-# Execute all of the automated tests related to Zcash.
-#
-
-set -eu
-
-SUITE_EXIT_STATUS=0
-REPOROOT="$(readlink -f "$(dirname "$0")"/../../)"
-
-function run_test_phase
-{
-    echo "===== BEGIN: $*"
-    set +e
-    eval "$@"
-    if [ $? -eq 0 ]
-    then
-        echo "===== PASSED: $*"
-    else
-        echo "===== FAILED: $*"
-        SUITE_EXIT_STATUS=1
-    fi
-    set -e
-}
-
-cd "${REPOROOT}"
-
-# Test phases:
-run_test_phase "${REPOROOT}/qa/zcash/check-security-hardening.sh"
-run_test_phase "${REPOROOT}/qa/zcash/ensure-no-dot-so-in-depends.py"
-
-# If make check fails, show test-suite.log as part of our run_test_phase
-# output (and fail the phase with false):
-run_test_phase make check '||' \
-               '{' \
-               echo '=== ./src/test-suite.log ===' ';' \
-               cat './src/test-suite.log' ';' \
-               false ';' \
-               '}'
-
-exit $SUITE_EXIT_STATUS
-
-
-
-
-
-
diff --git a/qa/zcash/full_test_suite.py b/qa/zcash/full_test_suite.py
new file mode 100755
index 000000000..f383d64e1
--- /dev/null
+++ b/qa/zcash/full_test_suite.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python2
+#
+# Execute all of the automated tests related to Zcash.
+#
+
+import argparse
+import os
+import subprocess
+import sys
+
+REPOROOT = os.path.dirname(
+    os.path.dirname(
+        os.path.dirname(
+            os.path.abspath(__file__)
+        )
+    )
+)
+
+def repofile(filename):
+    return os.path.join(REPOROOT, filename)
+
+
+#
+# Tests
+#
+
+STAGES = [
+    'btest',
+    'gtest',
+    'sec-hard',
+    'no-dot-so',
+    'secp256k1',
+    'univalue',
+    'rpc',
+]
+
+STAGE_COMMANDS = {
+    'btest': [repofile('src/test/test_bitcoin'), '-p'],
+    'gtest': [repofile('src/zcash-gtest')],
+    'sec-hard': [repofile('qa/zcash/check-security-hardening.sh')],
+    'no-dot-so': [repofile('qa/zcash/ensure-no-dot-so-in-depends.py')],
+    'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
+    'univalue': ['make', '-C', repofile('src/univalue'), 'check'],
+    'rpc': [repofile('qa/pull-tester/rpc-tests.sh')],
+}
+
+
+#
+# Test driver
+#
+
+def run_stage(stage):
+    print('Running stage %s' % stage)
+    print('=' * (len(stage) + 14))
+    print
+
+    ret = subprocess.call(STAGE_COMMANDS[stage])
+
+    print
+    print('-' * (len(stage) + 15))
+    print('Finished stage %s' % stage)
+    print
+
+    return ret == 0
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('stage', nargs='*', default=STAGES,
+                        help='One of %s'%STAGES)
+    args = parser.parse_args()
+
+    # Check validity of stages
+    for s in args.stage:
+        if s not in STAGES:
+            print("Invalid stage '%s' (choose from %s)" % (s, STAGES))
+            sys.exit(1)
+
+    # Run the stages
+    passed = True
+    for s in args.stage:
+        passed &= run_stage(s)
+
+    if not passed:
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

From 105b2b6248a982ed6cab5d141cf2672d09442d5b Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Fri, 6 Oct 2017 00:39:05 +0100
Subject: [PATCH 2/5] Move ensure-no-dot-so-in-depends.py into
 full_test_suite.py

---
 qa/zcash/ensure-no-dot-so-in-depends.py | 41 ----------------------
 qa/zcash/full_test_suite.py             | 45 +++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 44 deletions(-)
 delete mode 100755 qa/zcash/ensure-no-dot-so-in-depends.py

diff --git a/qa/zcash/ensure-no-dot-so-in-depends.py b/qa/zcash/ensure-no-dot-so-in-depends.py
deleted file mode 100755
index beb4b9ec0..000000000
--- a/qa/zcash/ensure-no-dot-so-in-depends.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#! /usr/bin/env python2
-
-import sys
-import os
-
-def main():
-    this_script = os.path.abspath(sys.argv[0])
-    basedir = os.path.dirname(this_script)
-    arch_dir = os.path.join(
-        basedir,
-        '..',
-        '..',
-        'depends',
-        'x86_64-unknown-linux-gnu',
-    )
-
-    exit_code = 0
-
-    if os.path.isdir(arch_dir):
-        lib_dir = os.path.join(arch_dir, 'lib')
-        libraries = os.listdir(lib_dir)
-
-        for lib in libraries:
-            if lib.find(".so") != -1:
-                print lib
-                exit_code = 1
-    else:
-        exit_code = 2
-        print "arch-specific build dir not present: {}".format(arch_dir)
-        print "Did you build the ./depends tree?"
-        print "Are you on a currently unsupported architecture?"
-
-    if exit_code == 0:
-        print "PASS."
-    else:
-        print "FAIL."
-
-    sys.exit(exit_code)
-
-if __name__ == '__main__':
-    main()
diff --git a/qa/zcash/full_test_suite.py b/qa/zcash/full_test_suite.py
index f383d64e1..d2eca6a42 100755
--- a/qa/zcash/full_test_suite.py
+++ b/qa/zcash/full_test_suite.py
@@ -20,6 +20,41 @@ def repofile(filename):
     return os.path.join(REPOROOT, filename)
 
 
+#
+# Custom test runners
+#
+
+def ensure_no_dot_so_in_depends():
+    arch_dir = os.path.join(
+        REPOROOT,
+        'depends',
+        'x86_64-unknown-linux-gnu',
+    )
+
+    exit_code = 0
+
+    if os.path.isdir(arch_dir):
+        lib_dir = os.path.join(arch_dir, 'lib')
+        libraries = os.listdir(lib_dir)
+
+        for lib in libraries:
+            if lib.find(".so") != -1:
+                print lib
+                exit_code = 1
+    else:
+        exit_code = 2
+        print "arch-specific build dir not present: {}".format(arch_dir)
+        print "Did you build the ./depends tree?"
+        print "Are you on a currently unsupported architecture?"
+
+    if exit_code == 0:
+        print "PASS."
+    else:
+        print "FAIL."
+
+    return exit_code == 0
+
+
 #
 # Tests
 #
@@ -38,7 +73,7 @@ STAGE_COMMANDS = {
     'btest': [repofile('src/test/test_bitcoin'), '-p'],
     'gtest': [repofile('src/zcash-gtest')],
     'sec-hard': [repofile('qa/zcash/check-security-hardening.sh')],
-    'no-dot-so': [repofile('qa/zcash/ensure-no-dot-so-in-depends.py')],
+    'no-dot-so': ensure_no_dot_so_in_depends,
     'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
     'univalue': ['make', '-C', repofile('src/univalue'), 'check'],
     'rpc': [repofile('qa/pull-tester/rpc-tests.sh')],
@@ -54,14 +89,18 @@ def run_stage(stage):
     print('=' * (len(stage) + 14))
     print
 
-    ret = subprocess.call(STAGE_COMMANDS[stage])
+    cmd = STAGE_COMMANDS[stage]
+    if type(cmd) == type([]):
+        ret = subprocess.call(cmd) == 0
+    else:
+        ret = cmd()
 
     print
     print('-' * (len(stage) + 15))
     print('Finished stage %s' % stage)
     print
 
-    return ret == 0
+    return ret
 
 def main():
     parser = argparse.ArgumentParser()

From c6af0aa45342a609c884f27446496b750f10a38f Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Fri, 6 Oct 2017 01:33:00 +0100
Subject: [PATCH 3/5] Move check-security-hardening.sh into full_test_suite.py

---
 qa/zcash/check-security-hardening.sh | 46 ----------------------
 qa/zcash/full_test_suite.py          | 58 +++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 47 deletions(-)
 delete mode 100755 qa/zcash/check-security-hardening.sh

diff --git a/qa/zcash/check-security-hardening.sh b/qa/zcash/check-security-hardening.sh
deleted file mode 100755
index 94a87fea3..000000000
--- a/qa/zcash/check-security-hardening.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -e
-
-REPOROOT="$(readlink -f "$(dirname "$0")"/../../)"
-
-function test_rpath_runpath {
-    if "${REPOROOT}/qa/zcash/checksec.sh" --file "$1" | grep -q "No RPATH.*No RUNPATH"; then
-        echo PASS: "$1" has no RPATH or RUNPATH.
-        return 0
-    else
-        echo FAIL: "$1" has an RPATH or a RUNPATH.
-        "${REPOROOT}/qa/zcash/checksec.sh" --file "$1"
-        return 1
-    fi
-}
-
-function test_fortify_source {
-    if { "${REPOROOT}/qa/zcash/checksec.sh" --fortify-file "$1" | grep -q "FORTIFY_SOURCE support available.*Yes"; } &&
-       { "${REPOROOT}/qa/zcash/checksec.sh" --fortify-file "$1" | grep -q "Binary compiled with FORTIFY_SOURCE support.*Yes"; }; then
-        echo PASS: "$1" has FORTIFY_SOURCE.
-        return 0
-    else
-        echo FAIL: "$1" is missing FORTIFY_SOURCE.
-        return 1
-    fi
-}
-
-# PIE, RELRO, Canary, and NX are tested by make check-security.
-make -C "$REPOROOT/src" check-security
-
-test_rpath_runpath "${REPOROOT}/src/zcashd"
-test_rpath_runpath "${REPOROOT}/src/zcash-cli"
-test_rpath_runpath "${REPOROOT}/src/zcash-gtest"
-test_rpath_runpath "${REPOROOT}/src/zcash-tx"
-test_rpath_runpath "${REPOROOT}/src/test/test_bitcoin"
-test_rpath_runpath "${REPOROOT}/src/zcash/GenerateParams"
-
-# NOTE: checksec.sh does not reliably determine whether FORTIFY_SOURCE is
-# enabled for the entire binary. See issue #915.
-test_fortify_source "${REPOROOT}/src/zcashd"
-test_fortify_source "${REPOROOT}/src/zcash-cli"
-test_fortify_source "${REPOROOT}/src/zcash-gtest"
-test_fortify_source "${REPOROOT}/src/zcash-tx"
-test_fortify_source "${REPOROOT}/src/test/test_bitcoin"
-test_fortify_source "${REPOROOT}/src/zcash/GenerateParams"
diff --git a/qa/zcash/full_test_suite.py b/qa/zcash/full_test_suite.py
index d2eca6a42..6e890c49a 100755
--- a/qa/zcash/full_test_suite.py
+++ b/qa/zcash/full_test_suite.py
@@ -5,6 +5,7 @@
 
 import argparse
 import os
+import re
 import subprocess
 import sys
 
@@ -24,6 +25,61 @@ def repofile(filename):
 # Custom test runners
 #
 
+RE_RPATH_RUNPATH = re.compile('No RPATH.*No RUNPATH')
+RE_FORTIFY_AVAILABLE = re.compile('FORTIFY_SOURCE support available.*Yes')
+RE_FORTIFY_USED = re.compile('Binary compiled with FORTIFY_SOURCE support.*Yes')
+
+def test_rpath_runpath(filename):
+    output = subprocess.check_output(
+        [repofile('qa/zcash/checksec.sh'), '--file', repofile(filename)]
+    )
+    if RE_RPATH_RUNPATH.search(output):
+        print('PASS: %s has no RPATH or RUNPATH.' % filename)
+        return True
+    else:
+        print('FAIL: %s has an RPATH or a RUNPATH.' % filename)
+        print(output)
+        return False
+
+def test_fortify_source(filename):
+    proc = subprocess.Popen(
+        [repofile('qa/zcash/checksec.sh'), '--fortify-file', repofile(filename)],
+        stdout=subprocess.PIPE,
+    )
+    line1 = proc.stdout.readline()
+    line2 = proc.stdout.readline()
+    proc.terminate()
+    if RE_FORTIFY_AVAILABLE.search(line1) and RE_FORTIFY_USED.search(line2):
+        print('PASS: %s has FORTIFY_SOURCE.' % filename)
+        return True
+    else:
+        print('FAIL: %s is missing FORTIFY_SOURCE.' % filename)
+        return False
+
+def check_security_hardening():
+    ret = True
+
+    # PIE, RELRO, Canary, and NX are tested by make check-security.
+    ret &= subprocess.call(['make', '-C', repofile('src'), 'check-security']) == 0
+
+    ret &= test_rpath_runpath('src/zcashd')
+    ret &= test_rpath_runpath('src/zcash-cli')
+    ret &= test_rpath_runpath('src/zcash-gtest')
+    ret &= test_rpath_runpath('src/zcash-tx')
+    ret &= test_rpath_runpath('src/test/test_bitcoin')
+    ret &= test_rpath_runpath('src/zcash/GenerateParams')
+
+    # NOTE: checksec.sh does not reliably determine whether FORTIFY_SOURCE
+    # is enabled for the entire binary. See issue #915.
+    ret &= test_fortify_source('src/zcashd')
+    ret &= test_fortify_source('src/zcash-cli')
+    ret &= test_fortify_source('src/zcash-gtest')
+    ret &= test_fortify_source('src/zcash-tx')
+    ret &= test_fortify_source('src/test/test_bitcoin')
+    ret &= test_fortify_source('src/zcash/GenerateParams')
+
+    return ret
+
 def ensure_no_dot_so_in_depends():
     arch_dir = os.path.join(
         REPOROOT,
@@ -72,7 +128,7 @@ STAGES = [
 STAGE_COMMANDS = {
     'btest': [repofile('src/test/test_bitcoin'), '-p'],
     'gtest': [repofile('src/zcash-gtest')],
-    'sec-hard': [repofile('qa/zcash/check-security-hardening.sh')],
+    'sec-hard': check_security_hardening,
     'no-dot-so': ensure_no_dot_so_in_depends,
     'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
     'univalue': ['make', '-C', repofile('src/univalue'), 'check'],

From 88fbdc48688f6d3f3bfaa5ae46a54e08e3571e83 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Mon, 16 Oct 2017 16:15:10 -0400
Subject: [PATCH 4/5] Add bitcoin-util-test.py to full_test_suite.py

Not moved, because upstream makes improvements to this script, and the need to
set environment variables makes it simpler to just use the given script.
---
 qa/zcash/full_test_suite.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/qa/zcash/full_test_suite.py b/qa/zcash/full_test_suite.py
index 6e890c49a..ecdd3a39a 100755
--- a/qa/zcash/full_test_suite.py
+++ b/qa/zcash/full_test_suite.py
@@ -110,6 +110,13 @@ def ensure_no_dot_so_in_depends():
 
     return exit_code == 0
 
+def util_test():
+    return subprocess.call(
+        [repofile('src/test/bitcoin-util-test.py')],
+        cwd=repofile('src'),
+        env={'PYTHONPATH': repofile('src/test'), 'srcdir': repofile('src')}
+    ) == 0
+
 
 #
 # Tests
@@ -120,6 +127,7 @@ STAGES = [
     'gtest',
     'sec-hard',
     'no-dot-so',
+    'util-test',
     'secp256k1',
     'univalue',
     'rpc',
@@ -130,6 +138,7 @@ STAGE_COMMANDS = {
     'gtest': [repofile('src/zcash-gtest')],
     'sec-hard': check_security_hardening,
     'no-dot-so': ensure_no_dot_so_in_depends,
+    'util-test': util_test,
     'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
     'univalue': ['make', '-C', repofile('src/univalue'), 'check'],
     'rpc': [repofile('qa/pull-tester/rpc-tests.sh')],

From 6e98511cf1498d7fa445d95a74bd09a60ff95a30 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Mon, 16 Oct 2017 16:16:14 -0400
Subject: [PATCH 5/5] Add stdout notice if any stage fails

---
 qa/zcash/full_test_suite.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/zcash/full_test_suite.py b/qa/zcash/full_test_suite.py
index ecdd3a39a..e01845635 100755
--- a/qa/zcash/full_test_suite.py
+++ b/qa/zcash/full_test_suite.py
@@ -185,6 +185,7 @@ def main():
         passed &= run_stage(s)
 
     if not passed:
+        print("!!! One or more test stages failed !!!")
         sys.exit(1)
 
 if __name__ == '__main__':