rename

qa/hush/checksec.sh

@@ -0,0 +1,882 @@
+#!/bin/bash
+#
+# The BSD License (http://www.opensource.org/licenses/bsd-license.php) 
+# specifies the terms and conditions of use for checksec.sh:
+#
+# Copyright (c) 2009-2011, Tobias Klein.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without 
+# modification, are permitted provided that the following conditions 
+# are met:
+# 
+# * Redistributions of source code must retain the above copyright 
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright 
+#   notice, this list of conditions and the following disclaimer in 
+#   the documentation and/or other materials provided with the 
+#   distribution.
+# * Neither the name of Tobias Klein nor the name of trapkit.de may be 
+#   used to endorse or promote products derived from this software 
+#   without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
+# DAMAGE.
+#
+# Name    : checksec.sh
+# Version : 1.5
+# Author  : Tobias Klein
+# Date    : November 2011
+# Download: http://www.trapkit.de/tools/checksec.html
+# Changes : http://www.trapkit.de/tools/checksec_changes.txt
+#
+# Description:
+#
+# Modern Linux distributions offer some mitigation techniques to make it 
+# harder to exploit software vulnerabilities reliably. Mitigations such 
+# as RELRO, NoExecute (NX), Stack Canaries, Address Space Layout 
+# Randomization (ASLR) and Position Independent Executables (PIE) have 
+# made reliably exploiting any vulnerabilities that do exist far more 
+# challenging. The checksec.sh script is designed to test what *standard* 
+# Linux OS and PaX (http://pax.grsecurity.net/) security features are being 
+# used.
+#
+# As of version 1.3 the script also lists the status of various Linux kernel 
+# protection mechanisms.
+#
+# Credits:
+#
+# Thanks to Brad Spengler (grsecurity.net) for the PaX support.
+# Thanks to Jon Oberheide (jon.oberheide.org) for the kernel support.
+# Thanks to Ollie Whitehouse (Research In Motion) for rpath/runpath support.
+# 
+# Others that contributed to checksec.sh (in no particular order):
+#
+# Simon Ruderich, Denis Scherbakov, Stefan Kuttler, Radoslaw Madej,
+# Anthony G. Basile, Martin Vaeth and Brian Davis. 
+#
+
+# global vars
+have_readelf=1
+verbose=false
+
+# FORTIFY_SOURCE vars
+FS_end=_chk
+FS_cnt_total=0
+FS_cnt_checked=0
+FS_cnt_unchecked=0
+FS_chk_func_libc=0
+FS_functions=0
+FS_libc=0
+ 
+# version information
+version() {
+  echo "checksec v1.5, Tobias Klein, www.trapkit.de, November 2011"
+  echo 
+}
+
+# help
+help() {
+  echo "Usage: checksec [OPTION]"
+  echo
+  echo "Options:"
+  echo
+  echo "  --file <executable-file>"
+  echo "  --dir <directory> [-v]"
+  echo "  --proc <process name>"
+  echo "  --proc-all"
+  echo "  --proc-libs <process ID>"
+  echo "  --kernel"
+  echo "  --fortify-file <executable-file>"
+  echo "  --fortify-proc <process ID>"
+  echo "  --version"
+  echo "  --help"
+  echo
+  echo "For more information, see:"
+  echo "  http://www.trapkit.de/tools/checksec.html"
+  echo
+}
+
+# check if command exists
+command_exists () {
+  type $1  > /dev/null 2>&1;
+}
+
+# check if directory exists
+dir_exists () {
+  if [ -d $1 ] ; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# check user privileges
+root_privs () {
+  if [ $(/usr/bin/id -u) -eq 0 ] ; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# check if input is numeric
+isNumeric () {
+  echo "$@" | grep -q -v "[^0-9]"
+}
+
+# check if input is a string
+isString () {
+  echo "$@" | grep -q -v "[^A-Za-z]"
+}
+
+# check file(s)
+filecheck() {
+  # check for RELRO support
+  if readelf -l $1 2>/dev/null | grep -q 'GNU_RELRO'; then
+    if readelf -d $1 2>/dev/null | grep -q 'BIND_NOW'; then
+      echo -n -e '\033[32mFull RELRO   \033[m   '
+    else
+      echo -n -e '\033[33mPartial RELRO\033[m   '
+    fi
+  else
+    echo -n -e '\033[31mNo RELRO     \033[m   '
+  fi
+
+  # check for stack canary support
+  if readelf -s $1 2>/dev/null | grep -q '__stack_chk_fail'; then
+    echo -n -e '\033[32mCanary found   \033[m   '
+  else
+    echo -n -e '\033[31mNo canary found\033[m   '
+  fi
+
+  # check for NX support
+  if readelf -W -l $1 2>/dev/null | grep 'GNU_STACK' | grep -q 'RWE'; then
+    echo -n -e '\033[31mNX disabled\033[m   '
+  else
+    echo -n -e '\033[32mNX enabled \033[m   '
+  fi 
+
+  # check for PIE support
+  if readelf -h $1 2>/dev/null | grep -q 'Type:[[:space:]]*EXEC'; then
+    echo -n -e '\033[31mNo PIE       \033[m   '
+  elif readelf -h $1 2>/dev/null | grep -q 'Type:[[:space:]]*DYN'; then
+    if readelf -d $1 2>/dev/null | grep -q '(DEBUG)'; then
+      echo -n -e '\033[32mPIE enabled  \033[m   '
+    else   
+      echo -n -e '\033[33mDSO          \033[m   '
+    fi
+  else
+    echo -n -e '\033[33mNot an ELF file\033[m   '
+  fi 
+
+  # check for rpath / run path
+  if readelf -d $1 2>/dev/null | grep -q 'rpath'; then
+   echo -n -e '\033[31mRPATH    \033[m  '
+  else
+   echo -n -e '\033[32mNo RPATH \033[m  '
+  fi
+
+  if readelf -d $1 2>/dev/null | grep -q 'runpath'; then
+   echo -n -e '\033[31mRUNPATH    \033[m  '
+  else
+   echo -n -e '\033[32mNo RUNPATH \033[m  '
+  fi
+}
+
+# check process(es)
+proccheck() {
+  # check for RELRO support
+  if readelf -l $1/exe 2>/dev/null | grep -q 'Program Headers'; then
+    if readelf -l $1/exe 2>/dev/null | grep -q 'GNU_RELRO'; then
+      if readelf -d $1/exe 2>/dev/null | grep -q 'BIND_NOW'; then
+        echo -n -e '\033[32mFull RELRO       \033[m '
+      else
+        echo -n -e '\033[33mPartial RELRO    \033[m '
+      fi
+    else
+      echo -n -e '\033[31mNo RELRO         \033[m '
+    fi
+  else
+    echo -n -e '\033[31mPermission denied (please run as root)\033[m\n'
+    exit 1
+  fi
+
+  # check for stack canary support
+  if readelf -s $1/exe 2>/dev/null | grep -q 'Symbol table'; then
+    if readelf -s $1/exe 2>/dev/null | grep -q '__stack_chk_fail'; then
+      echo -n -e '\033[32mCanary found         \033[m  '
+    else
+      echo -n -e '\033[31mNo canary found      \033[m  '
+    fi
+  else
+    if [ "$1" != "1" ] ; then
+      echo -n -e '\033[33mPermission denied    \033[m  '
+    else
+      echo -n -e '\033[33mNo symbol table found\033[m  '
+    fi
+  fi
+
+  # first check for PaX support
+  if cat $1/status 2> /dev/null | grep -q 'PaX:'; then
+    pageexec=( $(cat $1/status 2> /dev/null | grep 'PaX:' | cut -b6) )
+    segmexec=( $(cat $1/status 2> /dev/null | grep 'PaX:' | cut -b10) )
+    mprotect=( $(cat $1/status 2> /dev/null | grep 'PaX:' | cut -b8) )
+    randmmap=( $(cat $1/status 2> /dev/null | grep 'PaX:' | cut -b9) )
+    if [[ "$pageexec" = "P" || "$segmexec" = "S" ]] && [[ "$mprotect" = "M" && "$randmmap" = "R" ]] ; then
+      echo -n -e '\033[32mPaX enabled\033[m   '
+    elif [[ "$pageexec" = "p" && "$segmexec" = "s" && "$randmmap" = "R" ]] ; then
+      echo -n -e '\033[33mPaX ASLR only\033[m '
+    elif [[ "$pageexec" = "P" || "$segmexec" = "S" ]] && [[ "$mprotect" = "m" && "$randmmap" = "R" ]] ; then
+      echo -n -e '\033[33mPaX mprot off \033[m'
+    elif [[ "$pageexec" = "P" || "$segmexec" = "S" ]] && [[ "$mprotect" = "M" && "$randmmap" = "r" ]] ; then
+      echo -n -e '\033[33mPaX ASLR off\033[m  '
+    elif [[ "$pageexec" = "P" || "$segmexec" = "S" ]] && [[ "$mprotect" = "m" && "$randmmap" = "r" ]] ; then
+      echo -n -e '\033[33mPaX NX only\033[m   '
+    else
+      echo -n -e '\033[31mPaX disabled\033[m  '
+    fi
+  # fallback check for NX support
+  elif readelf -W -l $1/exe 2>/dev/null | grep 'GNU_STACK' | grep -q 'RWE'; then
+    echo -n -e '\033[31mNX disabled\033[m   '
+  else
+    echo -n -e '\033[32mNX enabled \033[m   '
+  fi 
+
+  # check for PIE support
+  if readelf -h $1/exe 2>/dev/null | grep -q 'Type:[[:space:]]*EXEC'; then
+    echo -n -e '\033[31mNo PIE               \033[m   '
+  elif readelf -h $1/exe 2>/dev/null | grep -q 'Type:[[:space:]]*DYN'; then
+    if readelf -d $1/exe 2>/dev/null | grep -q '(DEBUG)'; then
+      echo -n -e '\033[32mPIE enabled          \033[m   '
+    else   
+      echo -n -e '\033[33mDynamic Shared Object\033[m   '
+    fi
+  else
+    echo -n -e '\033[33mNot an ELF file      \033[m   '
+  fi
+}
+
+# check mapped libraries
+libcheck() {
+  libs=( $(awk '{ print $6 }' /proc/$1/maps | grep '/' | sort -u | xargs file | grep ELF | awk '{ print $1 }' | sed 's/:/ /') )
+ 
+  printf "\n* Loaded libraries (file information, # of mapped files: ${#libs[@]}):\n\n"
+  
+  for element in $(seq 0 $((${#libs[@]} - 1)))
+  do
+    echo "  ${libs[$element]}:"
+    echo -n "    "
+    filecheck ${libs[$element]}
+    printf "\n\n"
+  done
+}
+
+# check for system-wide ASLR support
+aslrcheck() {
+  # PaX ASLR support
+  if !(cat /proc/1/status 2> /dev/null | grep -q 'Name:') ; then
+    echo -n -e ':\033[33m insufficient privileges for PaX ASLR checks\033[m\n'
+    echo -n -e '  Fallback to standard Linux ASLR check'
+  fi
+  
+  if cat /proc/1/status 2> /dev/null | grep -q 'PaX:'; then
+    printf ": "
+    if cat /proc/1/status 2> /dev/null | grep 'PaX:' | grep -q 'R'; then
+      echo -n -e '\033[32mPaX ASLR enabled\033[m\n\n'
+    else
+      echo -n -e '\033[31mPaX ASLR disabled\033[m\n\n'
+    fi
+  else
+    # standard Linux 'kernel.randomize_va_space' ASLR support
+    # (see the kernel file 'Documentation/sysctl/kernel.txt' for a detailed description)
+    printf " (kernel.randomize_va_space): "
+    if /sbin/sysctl -a 2>/dev/null | grep -q 'kernel.randomize_va_space = 1'; then
+      echo -n -e '\033[33mOn (Setting: 1)\033[m\n\n'
+      printf "  Description - Make the addresses of mmap base, stack and VDSO page randomized.\n"
+      printf "  This, among other things, implies that shared libraries will be loaded to \n"
+      printf "  random addresses. Also for PIE-linked binaries, the location of code start\n"
+      printf "  is randomized. Heap addresses are *not* randomized.\n\n"
+    elif /sbin/sysctl -a 2>/dev/null | grep -q 'kernel.randomize_va_space = 2'; then
+      echo -n -e '\033[32mOn (Setting: 2)\033[m\n\n'
+      printf "  Description - Make the addresses of mmap base, heap, stack and VDSO page randomized.\n"
+      printf "  This, among other things, implies that shared libraries will be loaded to random \n"
+      printf "  addresses. Also for PIE-linked binaries, the location of code start is randomized.\n\n"
+    elif /sbin/sysctl -a 2>/dev/null | grep -q 'kernel.randomize_va_space = 0'; then
+      echo -n -e '\033[31mOff (Setting: 0)\033[m\n'
+    else
+      echo -n -e '\033[31mNot supported\033[m\n'
+    fi
+    printf "  See the kernel file 'Documentation/sysctl/kernel.txt' for more details.\n\n"
+  fi 
+}
+
+# check cpu nx flag
+nxcheck() {
+  if grep -q nx /proc/cpuinfo; then
+    echo -n -e '\033[32mYes\033[m\n\n'
+  else
+    echo -n -e '\033[31mNo\033[m\n\n'
+  fi
+}
+
+# check for kernel protection mechanisms
+kernelcheck() {
+  printf "  Description - List the status of kernel protection mechanisms. Rather than\n"
+  printf "  inspect kernel mechanisms that may aid in the prevention of exploitation of\n"
+  printf "  userspace processes, this option lists the status of kernel configuration\n"
+  printf "  options that harden the kernel itself against attack.\n\n"
+  printf "  Kernel config: "
+ 
+  if [ -f /proc/config.gz ] ; then
+    kconfig="zcat /proc/config.gz"
+    printf "\033[32m/proc/config.gz\033[m\n\n"
+  elif [ -f /boot/config-`uname -r` ] ; then
+    kconfig="cat /boot/config-`uname -r`"
+    printf "\033[33m/boot/config-`uname -r`\033[m\n\n"
+    printf "  Warning: The config on disk may not represent running kernel config!\n\n";
+  elif [ -f "${KBUILD_OUTPUT:-/usr/src/linux}"/.config ] ; then
+    kconfig="cat ${KBUILD_OUTPUT:-/usr/src/linux}/.config"
+    printf "\033[33m%s\033[m\n\n" "${KBUILD_OUTPUT:-/usr/src/linux}/.config"
+    printf "  Warning: The config on disk may not represent running kernel config!\n\n";
+  else
+    printf "\033[31mNOT FOUND\033[m\n\n"
+    exit 0
+  fi
+
+  printf "  GCC stack protector support:            "
+  if $kconfig | grep -qi 'CONFIG_CC_STACKPROTECTOR=y'; then
+    printf "\033[32mEnabled\033[m\n"
+  else
+    printf "\033[31mDisabled\033[m\n"
+  fi
+
+  printf "  Strict user copy checks:                "
+  if $kconfig | grep -qi 'CONFIG_DEBUG_STRICT_USER_COPY_CHECKS=y'; then
+    printf "\033[32mEnabled\033[m\n"
+  else
+    printf "\033[31mDisabled\033[m\n"
+  fi
+
+  printf "  Enforce read-only kernel data:          "
+  if $kconfig | grep -qi 'CONFIG_DEBUG_RODATA=y'; then
+    printf "\033[32mEnabled\033[m\n"
+  else
+    printf "\033[31mDisabled\033[m\n"
+  fi
+  printf "  Restrict /dev/mem access:               "
+  if $kconfig | grep -qi 'CONFIG_STRICT_DEVMEM=y'; then
+    printf "\033[32mEnabled\033[m\n"
+  else
+    printf "\033[31mDisabled\033[m\n"
+  fi
+
+  printf "  Restrict /dev/kmem access:              "
+  if $kconfig | grep -qi 'CONFIG_DEVKMEM=y'; then
+    printf "\033[31mDisabled\033[m\n"
+  else
+    printf "\033[32mEnabled\033[m\n"
+  fi
+
+  printf "\n"
+  printf "* grsecurity / PaX: "
+
+  if $kconfig | grep -qi 'CONFIG_GRKERNSEC=y'; then
+    if $kconfig | grep -qi 'CONFIG_GRKERNSEC_HIGH=y'; then
+      printf "\033[32mHigh GRKERNSEC\033[m\n\n"
+    elif $kconfig | grep -qi 'CONFIG_GRKERNSEC_MEDIUM=y'; then
+      printf "\033[33mMedium GRKERNSEC\033[m\n\n"
+    elif $kconfig | grep -qi 'CONFIG_GRKERNSEC_LOW=y'; then
+      printf "\033[31mLow GRKERNSEC\033[m\n\n"
+    else
+      printf "\033[33mCustom GRKERNSEC\033[m\n\n"
+    fi
+
+    printf "  Non-executable kernel pages:            "
+    if $kconfig | grep -qi 'CONFIG_PAX_KERNEXEC=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Prevent userspace pointer deref:        "
+    if $kconfig | grep -qi 'CONFIG_PAX_MEMORY_UDEREF=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Prevent kobject refcount overflow:      "
+    if $kconfig | grep -qi 'CONFIG_PAX_REFCOUNT=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Bounds check heap object copies:        "
+    if $kconfig | grep -qi 'CONFIG_PAX_USERCOPY=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Disable writing to kmem/mem/port:       "
+    if $kconfig | grep -qi 'CONFIG_GRKERNSEC_KMEM=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Disable privileged I/O:                 "
+    if $kconfig | grep -qi 'CONFIG_GRKERNSEC_IO=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Harden module auto-loading:             "
+    if $kconfig | grep -qi 'CONFIG_GRKERNSEC_MODHARDEN=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+
+    printf "  Hide kernel symbols:                    "
+    if $kconfig | grep -qi 'CONFIG_GRKERNSEC_HIDESYM=y'; then
+      printf "\033[32mEnabled\033[m\n"
+    else
+      printf "\033[31mDisabled\033[m\n"
+    fi
+  else
+    printf "\033[31mNo GRKERNSEC\033[m\n\n"
+    printf "  The grsecurity / PaX patchset is available here:\n"
+    printf "    http://grsecurity.net/\n"
+  fi
+
+  printf "\n"
+  printf "* Kernel Heap Hardening: "
+
+  if $kconfig | grep -qi 'CONFIG_KERNHEAP=y'; then
+    if $kconfig | grep -qi 'CONFIG_KERNHEAP_FULLPOISON=y'; then
+      printf "\033[32mFull KERNHEAP\033[m\n\n"
+    else
+      printf "\033[33mPartial KERNHEAP\033[m\n\n"
+    fi
+  else
+    printf "\033[31mNo KERNHEAP\033[m\n\n"
+    printf "  The KERNHEAP hardening patchset is available here:\n"
+    printf "    https://www.subreption.com/kernheap/\n\n"
+  fi
+}
+
+# --- FORTIFY_SOURCE subfunctions (start) ---
+
+# is FORTIFY_SOURCE supported by libc?
+FS_libc_check() {
+  printf "* FORTIFY_SOURCE support available (libc)    : "
+
+  if [ "${#FS_chk_func_libc[@]}" != "0" ] ; then
+    printf "\033[32mYes\033[m\n"
+  else
+    printf "\033[31mNo\033[m\n"
+    exit 1
+  fi
+}
+
+# was the binary compiled with FORTIFY_SOURCE?
+FS_binary_check() {
+  printf "* Binary compiled with FORTIFY_SOURCE support: "
+
+  for FS_elem_functions in $(seq 0 $((${#FS_functions[@]} - 1)))
+  do
+    if [[ ${FS_functions[$FS_elem_functions]} =~ _chk ]] ; then
+      printf "\033[32mYes\033[m\n"
+      return
+    fi
+  done
+  printf "\033[31mNo\033[m\n"
+  exit 1
+}
+
+FS_comparison() {
+  echo
+  printf " ------ EXECUTABLE-FILE ------- . -------- LIBC --------\n"
+  printf " FORTIFY-able library functions | Checked function names\n"
+  printf " -------------------------------------------------------\n"
+
+  for FS_elem_libc in $(seq 0 $((${#FS_chk_func_libc[@]} - 1)))
+  do
+    for FS_elem_functions in $(seq 0 $((${#FS_functions[@]} - 1)))
+    do
+      FS_tmp_func=${FS_functions[$FS_elem_functions]}
+      FS_tmp_libc=${FS_chk_func_libc[$FS_elem_libc]}
+
+      if [[ $FS_tmp_func =~ ^$FS_tmp_libc$ ]] ; then
+        printf " \033[31m%-30s\033[m | __%s%s\n" $FS_tmp_func $FS_tmp_libc $FS_end
+        let FS_cnt_total++
+        let FS_cnt_unchecked++
+      elif [[ $FS_tmp_func =~ ^$FS_tmp_libc(_chk) ]] ; then
+        printf " \033[32m%-30s\033[m | __%s%s\n" $FS_tmp_func $FS_tmp_libc $FS_end
+        let FS_cnt_total++
+        let FS_cnt_checked++
+      fi
+
+    done
+  done
+}
+
+FS_summary() {
+  echo
+  printf "SUMMARY:\n\n"
+  printf "* Number of checked functions in libc                : ${#FS_chk_func_libc[@]}\n"
+  printf "* Total number of library functions in the executable: ${#FS_functions[@]}\n"
+  printf "* Number of FORTIFY-able functions in the executable : %s\n" $FS_cnt_total
+  printf "* Number of checked functions in the executable      : \033[32m%s\033[m\n" $FS_cnt_checked
+  printf "* Number of unchecked functions in the executable    : \033[31m%s\033[m\n" $FS_cnt_unchecked
+  echo
+}
+
+# --- FORTIFY_SOURCE subfunctions (end) ---
+
+if !(command_exists readelf) ; then
+  printf "\033[31mWarning: 'readelf' not found! It's required for most checks.\033[m\n\n"
+  have_readelf=0
+fi
+
+# parse command-line arguments
+case "$1" in
+
+ --version)
+  version
+  exit 0
+  ;;
+
+ --help)
+  help
+  exit 0
+  ;;
+
+ --dir)
+  if [ "$3" = "-v" ] ; then
+    verbose=true
+  fi
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid directory.\033[m\n\n"
+    exit 1
+  fi
+  # remove trailing slashes
+  tempdir=`echo $2 | sed -e "s/\/*$//"`
+  if [ ! -d $tempdir ] ; then
+    printf "\033[31mError: The directory '$tempdir' does not exist.\033[m\n\n"
+    exit 1
+  fi
+  cd $tempdir
+  printf "RELRO           STACK CANARY      NX            PIE             RPATH      RUNPATH      FILE\n"
+  for N in [A-Za-z]*; do
+    if [ "$N" != "[A-Za-z]*" ]; then
+      # read permissions?
+      if [ ! -r $N ]; then
+        printf "\033[31mError: No read permissions for '$tempdir/$N' (run as root).\033[m\n"
+      else
+        # ELF executable?
+        out=`file $N`
+        if [[ ! $out =~ ELF ]] ; then
+           if [ "$verbose" = "true" ] ; then
+             printf "\033[34m*** Not an ELF file: $tempdir/"
+             file $N
+             printf "\033[m"
+           fi
+        else 
+          filecheck $N
+          if [ `find $tempdir/$N \( -perm -004000 -o -perm -002000 \) -type f -print` ]; then
+            printf "\033[37;41m%s%s\033[m" $2 $N
+          else
+            printf "%s%s" $tempdir/ $N
+          fi
+          echo
+        fi
+      fi
+    fi
+  done
+  exit 0
+  ;;
+ 
+ --file)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid file.\033[m\n\n"
+   exit 1
+  fi
+  # does the file exist?
+  if [ ! -e $2 ] ; then
+    printf "\033[31mError: The file '$2' does not exist.\033[m\n\n"
+    exit 1
+  fi
+  # read permissions?
+  if [ ! -r $2 ] ; then
+    printf "\033[31mError: No read permissions for '$2' (run as root).\033[m\n\n"
+    exit 1
+  fi
+  # ELF executable?
+  out=`file $2`
+  if [[ ! $out =~ ELF ]] ; then
+    printf "\033[31mError: Not an ELF file: "
+    file $2
+    printf "\033[m\n"
+    exit 1
+  fi
+  printf "RELRO           STACK CANARY      NX            PIE             RPATH      RUNPATH      FILE\n"
+  filecheck $2
+  if [ `find $2 \( -perm -004000 -o -perm -002000 \) -type f -print` ] ; then
+    printf "\033[37;41m%s%s\033[m" $2 $N
+  else
+    printf "%s" $2
+  fi
+  echo
+  exit 0
+  ;;
+
+ --proc-all)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  cd /proc
+  printf "* System-wide ASLR"
+  aslrcheck
+  printf "* Does the CPU support NX: "
+  nxcheck 
+  printf "         COMMAND    PID RELRO             STACK CANARY           NX/PaX        PIE\n"
+  for N in [1-9]*; do
+    if [ $N != $$ ] && readlink -q $N/exe > /dev/null; then
+      printf "%16s" `head -1 $N/status | cut -b 7-`
+      printf "%7d " $N
+      proccheck $N
+      echo
+    fi
+  done
+  if [ ! -e /usr/bin/id ] ; then
+    printf "\n\033[33mNote: If you are running 'checksec.sh' as an unprivileged user, you\n"
+    printf "      will not see all processes. Please run the script as root.\033[m\n\n"
+  else 
+    if !(root_privs) ; then
+      printf "\n\033[33mNote: You are running 'checksec.sh' as an unprivileged user.\n" 
+      printf "      Too see all processes, please run the script as root.\033[m\n\n"
+    fi
+  fi
+  exit 0
+  ;;
+
+ --proc)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid process name.\033[m\n\n"
+    exit 1
+  fi
+  if !(isString "$2") ; then
+     printf "\033[31mError: Please provide a valid process name.\033[m\n\n"
+     exit 1
+  fi
+  cd /proc
+  printf "* System-wide ASLR"
+  aslrcheck
+  printf "* Does the CPU support NX: "
+  nxcheck
+  printf "         COMMAND    PID RELRO             STACK CANARY           NX/PaX        PIE\n"
+  for N in `ps -Ao pid,comm | grep $2 | cut -b1-6`; do
+    if [ -d $N ] ; then
+      printf "%16s" `head -1 $N/status | cut -b 7-`
+      printf "%7d " $N
+      # read permissions?
+      if [ ! -r $N/exe ] ; then
+        if !(root_privs) ; then
+          printf "\033[31mNo read permissions for '/proc/$N/exe' (run as root).\033[m\n\n"
+          exit 1
+        fi
+        if [ ! `readlink $N/exe` ] ; then
+          printf "\033[31mPermission denied. Requested process ID belongs to a kernel thread.\033[m\n\n"
+          exit 1
+        fi
+        exit 1
+      fi
+      proccheck $N
+      echo
+    fi
+  done
+  exit 0
+  ;;
+
+ --proc-libs)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid process ID.\033[m\n\n"
+    exit 1
+  fi
+  if !(isNumeric "$2") ; then
+     printf "\033[31mError: Please provide a valid process ID.\033[m\n\n"
+     exit 1
+  fi
+  cd /proc
+  printf "* System-wide ASLR"
+  aslrcheck
+  printf "* Does the CPU support NX: "
+  nxcheck
+  printf "* Process information:\n\n"
+  printf "         COMMAND    PID RELRO             STACK CANARY           NX/PaX        PIE\n"
+  N=$2
+  if [ -d $N ] ; then
+    printf "%16s" `head -1 $N/status | cut -b 7-`
+    printf "%7d " $N
+    # read permissions?
+    if [ ! -r $N/exe ] ; then
+      if !(root_privs) ; then
+        printf "\033[31mNo read permissions for '/proc/$N/exe' (run as root).\033[m\n\n"
+        exit 1
+      fi
+      if [ ! `readlink $N/exe` ] ; then
+        printf "\033[31mPermission denied. Requested process ID belongs to a kernel thread.\033[m\n\n"
+        exit 1
+      fi
+      exit 1
+    fi
+    proccheck $N
+    echo
+    libcheck $N
+  fi
+  exit 0
+  ;;
+
+ --kernel)
+  cd /proc
+  printf "* Kernel protection information:\n\n"
+  kernelcheck
+  exit 0
+  ;;
+
+ --fortify-file)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid file.\033[m\n\n"
+   exit 1
+  fi
+  # does the file exist?
+  if [ ! -e $2 ] ; then
+    printf "\033[31mError: The file '$2' does not exist.\033[m\n\n"
+    exit 1
+  fi
+  # read permissions?
+  if [ ! -r $2 ] ; then
+    printf "\033[31mError: No read permissions for '$2' (run as root).\033[m\n\n"
+    exit 1
+  fi
+  # ELF executable?
+  out=`file $2`
+  if [[ ! $out =~ ELF ]] ; then
+    printf "\033[31mError: Not an ELF file: "
+    file $2
+    printf "\033[m\n"
+    exit 1
+  fi
+  if [ -e /lib/libc.so.6 ] ; then
+    FS_libc=/lib/libc.so.6
+  elif [ -e /lib64/libc.so.6 ] ; then
+    FS_libc=/lib64/libc.so.6
+  elif [ -e /lib/i386-linux-gnu/libc.so.6 ] ; then
+    FS_libc=/lib/i386-linux-gnu/libc.so.6
+  elif [ -e /lib/x86_64-linux-gnu/libc.so.6 ] ; then
+    FS_libc=/lib/x86_64-linux-gnu/libc.so.6
+  else
+    printf "\033[31mError: libc not found.\033[m\n\n"
+    exit 1
+  fi
+
+  FS_chk_func_libc=( $(readelf -s $FS_libc | grep _chk@@ | awk '{ print $8 }' | cut -c 3- | sed -e 's/_chk@.*//') )
+  FS_functions=( $(readelf -s $2 | awk '{ print $8 }' | sed 's/_*//' | sed -e 's/@.*//') )
+
+  FS_libc_check
+  FS_binary_check
+  FS_comparison
+  FS_summary
+
+  exit 0
+  ;;
+
+ --fortify-proc)
+  if [ $have_readelf -eq 0 ] ; then
+    exit 1
+  fi
+  if [ -z "$2" ] ; then
+    printf "\033[31mError: Please provide a valid process ID.\033[m\n\n"
+    exit 1
+  fi
+  if !(isNumeric "$2") ; then
+     printf "\033[31mError: Please provide a valid process ID.\033[m\n\n"
+     exit 1
+  fi
+  cd /proc
+  N=$2
+  if [ -d $N ] ; then
+    # read permissions?
+    if [ ! -r $N/exe ] ; then
+      if !(root_privs) ; then
+        printf "\033[31mNo read permissions for '/proc/$N/exe' (run as root).\033[m\n\n"
+        exit 1
+      fi
+      if [ ! `readlink $N/exe` ] ; then
+        printf "\033[31mPermission denied. Requested process ID belongs to a kernel thread.\033[m\n\n"
+        exit 1
+      fi
+      exit 1
+    fi
+    if [ -e /lib/libc.so.6 ] ; then
+      FS_libc=/lib/libc.so.6
+    elif [ -e /lib64/libc.so.6 ] ; then
+      FS_libc=/lib64/libc.so.6
+    elif [ -e /lib/i386-linux-gnu/libc.so.6 ] ; then
+      FS_libc=/lib/i386-linux-gnu/libc.so.6
+    elif [ -e /lib/x86_64-linux-gnu/libc.so.6 ] ; then
+      FS_libc=/lib/x86_64-linux-gnu/libc.so.6
+    else
+      printf "\033[31mError: libc not found.\033[m\n\n"
+      exit 1
+    fi
+    printf "* Process name (PID)                         : %s (%d)\n" `head -1 $N/status | cut -b 7-` $N
+    FS_chk_func_libc=( $(readelf -s $FS_libc | grep _chk@@ | awk '{ print $8 }' | cut -c 3- | sed -e 's/_chk@.*//') )
+    FS_functions=( $(readelf -s $2/exe | awk '{ print $8 }' | sed 's/_*//' | sed -e 's/@.*//') )
+
+    FS_libc_check
+    FS_binary_check
+    FS_comparison
+    FS_summary
+  fi
+  exit 0
+  ;;
+
+ *)
+  if [ "$#" != "0" ] ; then
+    printf "\033[31mError: Unknown option '$1'.\033[m\n\n"
+  fi
+  help
+  exit 1
+  ;;
+esac

qa/hush/create_benchmark_archive.py

@@ -0,0 +1,262 @@
+import binascii
+import calendar
+import json
+import plyvel
+import progressbar
+import os
+import stat
+import subprocess
+import sys
+import tarfile
+import time
+
+ZCASH_CLI = './src/zcash-cli'
+USAGE = """
+Requirements:
+- find
+- xz
+- %s (edit ZCASH_CLI in this script to alter the path)
+- A running mainnet zcashd using the default datadir with -txindex=1
+
+Example usage:
+
+make -C src/leveldb/
+virtualenv venv
+. venv/bin/activate
+pip install --global-option=build_ext --global-option="-L$(pwd)/src/leveldb/" --global-option="-I$(pwd)/src/leveldb/include/" plyvel
+pip install progressbar2
+LD_LIBRARY_PATH=src/leveldb python qa/zcash/create_benchmark_archive.py
+""" % ZCASH_CLI
+
+def check_deps():
+    if subprocess.call(['which', 'find', 'xz', ZCASH_CLI], stdout=subprocess.PIPE):
+        print USAGE
+        sys.exit()
+
+def encode_varint(n):
+    v = bytearray()
+    l = 0
+    while True:
+        v.append((n & 0x7F) | (0x80 if l else 0x00))
+        if (n <= 0x7F):
+            break
+        n = (n >> 7) - 1
+        l += 1
+    return bytes(v)[::-1]
+
+def decode_varint(v):
+    n = 0
+    for ch in range(len(v)):
+        n = (n << 7) | (ord(v[ch]) & 0x7F)
+        if (ord(v[ch]) & 0x80):
+            n += 1
+        else:
+            return n
+
+def compress_amount(n):
+    if n == 0:
+        return 0
+    e = 0
+    while (((n % 10) == 0) and e < 9):
+        n /= 10
+        e += 1
+    if e < 9:
+        d = (n % 10)
+        assert(d >= 1 and d <= 9)
+        n /= 10
+        return 1 + (n*9 + d - 1)*10 + e
+    else:
+        return 1 + (n - 1)*10 + 9
+
+OP_DUP = 0x76
+OP_EQUAL = 0x87
+OP_EQUALVERIFY = 0x88
+OP_HASH160 = 0xa9
+OP_CHECKSIG = 0xac
+def to_key_id(script):
+    if len(script) == 25 and \
+            script[0] == OP_DUP and \
+            script[1] == OP_HASH160 and \
+            script[2] == 20 and \
+            script[23] == OP_EQUALVERIFY and \
+            script[24] == OP_CHECKSIG:
+        return script[3:23]
+    return bytes()
+
+def to_script_id(script):
+    if len(script) == 23 and \
+            script[0] == OP_HASH160 and \
+            script[1] == 20 and \
+            script[22] == OP_EQUAL:
+        return script[2:22]
+    return bytes()
+
+def to_pubkey(script):
+    if len(script) == 35 and \
+            script[0] == 33 and \
+            script[34] == OP_CHECKSIG and \
+            (script[1] == 0x02 or script[1] == 0x03):
+        return script[1:34]
+    if len(script) == 67 and \
+            script[0] == 65 and \
+            script[66] == OP_CHECKSIG and \
+            script[1] == 0x04:
+        return script[1:66] # assuming is fully valid
+    return bytes()
+
+def compress_script(script):
+    result = bytearray()
+
+    key_id = to_key_id(script)
+    if key_id:
+        result.append(0x00)
+        result.extend(key_id)
+        return bytes(result)
+
+    script_id = to_script_id(script)
+    if script_id:
+        result.append(0x01)
+        result.extend(script_id)
+        return bytes(result)
+
+    pubkey = to_pubkey(script)
+    if pubkey:
+        result.append(0x00)
+        result.extend(pubkey[1:33])
+        if pubkey[0] == 0x02 or pubkey[0] == 0x03:
+            result[0] = pubkey[0]
+            return bytes(result)
+        elif pubkey[0] == 0x04:
+            result[0] = 0x04 | (pubkey[64] & 0x01)
+            return bytes(result)
+
+    size = len(script) + 6
+    result.append(encode_varint(size))
+    result.extend(script)
+    return bytes(result)
+
+def deterministic_filter(tarinfo):
+    tarinfo.uid = tarinfo.gid = 0
+    tarinfo.uname = tarinfo.gname = "root"
+    tarinfo.mtime = calendar.timegm(time.strptime('2017-05-17', '%Y-%m-%d'))
+    tarinfo.mode |= stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
+    tarinfo.mode &= ~stat.S_IWGRP
+    if tarinfo.isdir():
+        tarinfo.mode |= \
+            stat.S_IXUSR | \
+            stat.S_IXGRP | \
+            stat.S_IXOTH
+    else:
+        tarinfo.mode &= \
+            ~stat.S_IXUSR & \
+            ~stat.S_IXGRP & \
+            ~stat.S_IXOTH
+    return tarinfo
+
+def create_benchmark_archive(blk_hash):
+    blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash]))
+    print 'Height: %d' % blk['height']
+    print 'Transactions: %d' % len(blk['tx'])
+
+    os.mkdir('benchmark')
+    with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f:
+        f.write(binascii.unhexlify(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash, 'false']).strip()))
+
+    txs = [json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1'])
+                     ) for tx in blk['tx']]
+
+    js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0])
+    if js_txs:
+        print 'Block contains %d JoinSplit-containing transactions' % js_txs
+        return
+
+    inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin'] if x.has_key('txid')]
+    print 'Total inputs: %d' % len(inputs)
+
+    unique_inputs = {}
+    for i in sorted(inputs):
+        if unique_inputs.has_key(i[0]):
+            unique_inputs[i[0]].append(i[1])
+        else:
+            unique_inputs[i[0]] = [i[1]]
+    print 'Unique input transactions: %d' % len(unique_inputs)
+
+    db_path = 'benchmark/block-%d-inputs' % blk['height']
+    db = plyvel.DB(db_path, create_if_missing=True)
+    wb = db.write_batch()
+    bar = progressbar.ProgressBar(redirect_stdout=True)
+    print 'Collecting input coins for block'
+    for tx in bar(unique_inputs.keys()):
+        rawtx = json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1']))
+
+        mask_size = 0
+        mask_code = 0
+        b = 0
+        while 2+b*8 < len(rawtx['vout']):
+            zero = True
+            i = 0
+            while i < 8 and 2+b*8+i < len(rawtx['vout']):
+                if 2+b*8+i in unique_inputs[tx]:
+                    zero = False
+                i += 1
+            if not zero:
+                mask_size = b + 1
+                mask_code += 1
+            b += 1
+
+        coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0]
+        first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx]
+        second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx]
+        code = 8*(mask_code - (0 if first or second else 1)) + \
+            (1 if coinbase else 0) + \
+            (2 if first else 0) + \
+            (4 if second else 0)
+
+        coins = bytearray()
+        # Serialized format:
+        # - VARINT(nVersion)
+        coins.extend(encode_varint(rawtx['version']))
+        # - VARINT(nCode)
+        coins.extend(encode_varint(code))
+        # - unspentness bitvector, for vout[2] and further; least significant byte first
+        for b in range(mask_size):
+            avail = 0
+            i = 0
+            while i < 8 and 2+b*8+i < len(rawtx['vout']):
+                if 2+b*8+i in unique_inputs[tx]:
+                    avail |= (1 << i)
+                i += 1
+            coins.append(avail)
+        # - the non-spent CTxOuts (via CTxOutCompressor)
+        for i in range(len(rawtx['vout'])):
+            if i in unique_inputs[tx]:
+                coins.extend(encode_varint(compress_amount(int(rawtx['vout'][i]['valueZat']))))
+                coins.extend(compress_script(
+                    binascii.unhexlify(rawtx['vout'][i]['scriptPubKey']['hex'])))
+        # - VARINT(nHeight)
+        coins.extend(encode_varint(json.loads(
+            subprocess.check_output([ZCASH_CLI, 'getblockheader', rawtx['blockhash']])
+            )['height']))
+
+        db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1])
+        db_val = bytes(coins)
+        wb.put(db_key, db_val)
+
+    wb.write()
+    db.close()
+
+    # Make reproducible archive
+    os.remove('%s/LOG' % db_path)
+    files = subprocess.check_output(['find', 'benchmark']).strip().split('\n')
+    archive_name = 'block-%d.tar' % blk['height']
+    tar = tarfile.open(archive_name, 'w')
+    for name in sorted(files):
+        tar.add(name, recursive=False, filter=deterministic_filter)
+    tar.close()
+    subprocess.check_call(['xz', '-6', archive_name])
+    print 'Created archive %s.xz' % archive_name
+    subprocess.call(['rm', '-r', 'benchmark'])
+
+if __name__ == '__main__':
+    check_deps()
+    create_benchmark_archive('0000000007cdb809e48e51dd0b530e8f5073e0a9e9bd7ae920fe23e874658c74')

qa/hush/create_wallet_200k_utxos.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python2
+# Copyright (c) 2017 The Zcash developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Create a large wallet
+#
+# To use:
+# - Copy to qa/rpc-tests/wallet_large.py
+# - Add wallet_large.py to RPC tests list
+# - ./qa/pull-tester/rpc-tests.sh wallet_large --nocleanup
+# - Archive the resulting /tmp/test###### directory
+#
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    connect_nodes_bi,
+    initialize_chain_clean,
+    start_nodes,
+)
+
+from decimal import Decimal
+
+
+class LargeWalletTest(BitcoinTestFramework):
+
+    def setup_chain(self):
+        print("Initializing test directory "+self.options.tmpdir)
+        initialize_chain_clean(self.options.tmpdir, 2)
+
+    def setup_network(self):
+        self.nodes = start_nodes(2, self.options.tmpdir)
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.is_network_split = False
+        self.sync_all()
+
+    def run_test(self):
+        self.nodes[1].generate(103)
+        self.sync_all()
+
+        inputs = []
+        for i in range(200000):
+            taddr = self.nodes[0].getnewaddress()
+            inputs.append(self.nodes[1].sendtoaddress(taddr, Decimal("0.001")))
+            if i % 1000 == 0:
+                self.nodes[1].generate(1)
+                self.sync_all()
+
+        self.nodes[1].generate(1)
+        self.sync_all()
+        print('Node 0: %d transactions, %d UTXOs' %
+              (len(self.nodes[0].listtransactions()), len(self.nodes[0].listunspent())))
+        print('Node 1: %d transactions, %d UTXOs' %
+              (len(self.nodes[1].listtransactions()), len(self.nodes[1].listunspent())))
+        assert_equal(len(self.nodes[0].listunspent()), len(inputs))
+
+if __name__ == '__main__':
+    LargeWalletTest().main()

qa/hush/full-test-suite.sh

@@ -0,0 +1,201 @@
+#!/usr/bin/env python2
+#
+# Execute all of the automated tests related to Zcash.
+#
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+REPOROOT = os.path.dirname(
+    os.path.dirname(
+        os.path.dirname(
+            os.path.abspath(__file__)
+        )
+    )
+)
+
+def repofile(filename):
+    return os.path.join(REPOROOT, filename)
+
+
+#
+# Custom test runners
+#
+
+RE_RPATH_RUNPATH = re.compile('No RPATH.*No RUNPATH')
+RE_FORTIFY_AVAILABLE = re.compile('FORTIFY_SOURCE support available.*Yes')
+RE_FORTIFY_USED = re.compile('Binary compiled with FORTIFY_SOURCE support.*Yes')
+
+def test_rpath_runpath(filename):
+    output = subprocess.check_output(
+        [repofile('qa/zcash/checksec.sh'), '--file', repofile(filename)]
+    )
+    if RE_RPATH_RUNPATH.search(output):
+        print('PASS: %s has no RPATH or RUNPATH.' % filename)
+        return True
+    else:
+        print('FAIL: %s has an RPATH or a RUNPATH.' % filename)
+        print(output)
+        return False
+
+def test_fortify_source(filename):
+    proc = subprocess.Popen(
+        [repofile('qa/zcash/checksec.sh'), '--fortify-file', repofile(filename)],
+        stdout=subprocess.PIPE,
+    )
+    line1 = proc.stdout.readline()
+    line2 = proc.stdout.readline()
+    proc.terminate()
+    if RE_FORTIFY_AVAILABLE.search(line1) and RE_FORTIFY_USED.search(line2):
+        print('PASS: %s has FORTIFY_SOURCE.' % filename)
+        return True
+    else:
+        print('FAIL: %s is missing FORTIFY_SOURCE.' % filename)
+        return False
+
+def check_security_hardening():
+    ret = True
+
+    # PIE, RELRO, Canary, and NX are tested by make check-security.
+    ret &= subprocess.call(['make', '-C', repofile('src'), 'check-security']) == 0
+
+    ret &= test_rpath_runpath('src/zcashd')
+    ret &= test_rpath_runpath('src/zcash-cli')
+    ret &= test_rpath_runpath('src/zcash-gtest')
+    ret &= test_rpath_runpath('src/zcash-tx')
+    ret &= test_rpath_runpath('src/test/test_bitcoin')
+    ret &= test_rpath_runpath('src/zcash/GenerateParams')
+
+    # NOTE: checksec.sh does not reliably determine whether FORTIFY_SOURCE
+    # is enabled for the entire binary. See issue #915.
+    ret &= test_fortify_source('src/zcashd')
+    ret &= test_fortify_source('src/zcash-cli')
+    ret &= test_fortify_source('src/zcash-gtest')
+    ret &= test_fortify_source('src/zcash-tx')
+    ret &= test_fortify_source('src/test/test_bitcoin')
+    ret &= test_fortify_source('src/zcash/GenerateParams')
+
+    return ret
+
+def ensure_no_dot_so_in_depends():
+    arch_dir = os.path.join(
+        REPOROOT,
+        'depends',
+        'x86_64-unknown-linux-gnu',
+    )
+
+    exit_code = 0
+
+    if os.path.isdir(arch_dir):
+        lib_dir = os.path.join(arch_dir, 'lib')
+        libraries = os.listdir(lib_dir)
+
+        for lib in libraries:
+            if lib.find(".so") != -1:
+                print lib
+                exit_code = 1
+    else:
+        exit_code = 2
+        print "arch-specific build dir not present: {}".format(arch_dir)
+        print "Did you build the ./depends tree?"
+        print "Are you on a currently unsupported architecture?"
+
+    if exit_code == 0:
+        print "PASS."
+    else:
+        print "FAIL."
+
+    return exit_code == 0
+
+def util_test():
+    return subprocess.call(
+        [repofile('src/test/bitcoin-util-test.py')],
+        cwd=repofile('src'),
+        env={'PYTHONPATH': repofile('src/test'), 'srcdir': repofile('src')}
+    ) == 0
+
+
+#
+# Tests
+#
+
+STAGES = [
+    'btest',
+    'gtest',
+    'sec-hard',
+    'no-dot-so',
+    'util-test',
+    'secp256k1',
+    'libsnark',
+    'univalue',
+    'rpc',
+]
+
+STAGE_COMMANDS = {
+    'btest': [repofile('src/test/test_bitcoin'), '-p'],
+    'gtest': [repofile('src/zcash-gtest')],
+    'sec-hard': check_security_hardening,
+    'no-dot-so': ensure_no_dot_so_in_depends,
+    'util-test': util_test,
+    'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
+    'libsnark': ['make', '-C', repofile('src'), 'libsnark-tests'],
+    'univalue': ['make', '-C', repofile('src/univalue'), 'check'],
+    'rpc': [repofile('qa/pull-tester/rpc-tests.sh')],
+}
+
+
+#
+# Test driver
+#
+
+def run_stage(stage):
+    print('Running stage %s' % stage)
+    print('=' * (len(stage) + 14))
+    print
+
+    cmd = STAGE_COMMANDS[stage]
+    if type(cmd) == type([]):
+        ret = subprocess.call(cmd) == 0
+    else:
+        ret = cmd()
+
+    print
+    print('-' * (len(stage) + 15))
+    print('Finished stage %s' % stage)
+    print
+
+    return ret
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list-stages', dest='list', action='store_true')
+    parser.add_argument('stage', nargs='*', default=STAGES,
+                        help='One of %s'%STAGES)
+    args = parser.parse_args()
+
+    # Check for list
+    if args.list:
+        for s in STAGES:
+            print(s)
+        sys.exit(0)
+
+    # Check validity of stages
+    for s in args.stage:
+        if s not in STAGES:
+            print("Invalid stage '%s' (choose from %s)" % (s, STAGES))
+            sys.exit(1)
+
+    # Run the stages
+    passed = True
+    for s in args.stage:
+        passed &= run_stage(s)
+
+    if not passed:
+        print("!!! One or more test stages failed !!!")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

qa/hush/full_test_suite.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python2
+#
+# Execute all of the automated tests related to Zcash.
+#
+
+import argparse
+from glob import glob
+import os
+import re
+import subprocess
+import sys
+
+REPOROOT = os.path.dirname(
+    os.path.dirname(
+        os.path.dirname(
+            os.path.abspath(__file__)
+        )
+    )
+)
+
+def repofile(filename):
+    return os.path.join(REPOROOT, filename)
+
+
+#
+# Custom test runners
+#
+
+RE_RPATH_RUNPATH = re.compile('No RPATH.*No RUNPATH')
+RE_FORTIFY_AVAILABLE = re.compile('FORTIFY_SOURCE support available.*Yes')
+RE_FORTIFY_USED = re.compile('Binary compiled with FORTIFY_SOURCE support.*Yes')
+
+def test_rpath_runpath(filename):
+    output = subprocess.check_output(
+        [repofile('qa/zcash/checksec.sh'), '--file', repofile(filename)]
+    )
+    if RE_RPATH_RUNPATH.search(output):
+        print('PASS: %s has no RPATH or RUNPATH.' % filename)
+        return True
+    else:
+        print('FAIL: %s has an RPATH or a RUNPATH.' % filename)
+        print(output)
+        return False
+
+def test_fortify_source(filename):
+    proc = subprocess.Popen(
+        [repofile('qa/zcash/checksec.sh'), '--fortify-file', repofile(filename)],
+        stdout=subprocess.PIPE,
+    )
+    line1 = proc.stdout.readline()
+    line2 = proc.stdout.readline()
+    proc.terminate()
+    if RE_FORTIFY_AVAILABLE.search(line1) and RE_FORTIFY_USED.search(line2):
+        print('PASS: %s has FORTIFY_SOURCE.' % filename)
+        return True
+    else:
+        print('FAIL: %s is missing FORTIFY_SOURCE.' % filename)
+        return False
+
+def check_security_hardening():
+    ret = True
+
+    # PIE, RELRO, Canary, and NX are tested by make check-security.
+    ret &= subprocess.call(['make', '-C', repofile('src'), 'check-security']) == 0
+
+    # The remaining checks are only for ELF binaries
+    # Assume that if zcashd is an ELF binary, they all are
+    with open(repofile('src/zcashd'), 'rb') as f:
+        magic = f.read(4)
+        if not magic.startswith(b'\x7fELF'):
+            return ret
+
+    ret &= test_rpath_runpath('src/zcashd')
+    ret &= test_rpath_runpath('src/zcash-cli')
+    ret &= test_rpath_runpath('src/zcash-gtest')
+    ret &= test_rpath_runpath('src/zcash-tx')
+    ret &= test_rpath_runpath('src/test/test_bitcoin')
+
+    # NOTE: checksec.sh does not reliably determine whether FORTIFY_SOURCE
+    # is enabled for the entire binary. See issue #915.
+    ret &= test_fortify_source('src/zcashd')
+    ret &= test_fortify_source('src/zcash-cli')
+    ret &= test_fortify_source('src/zcash-gtest')
+    ret &= test_fortify_source('src/zcash-tx')
+    ret &= test_fortify_source('src/test/test_bitcoin')
+
+    return ret
+
+def ensure_no_dot_so_in_depends():
+    depends_dir = os.path.join(REPOROOT, 'depends')
+    arch_dir = os.path.join(depends_dir, 'x86_64-unknown-linux-gnu')
+    if not os.path.isdir(arch_dir):
+        # Not Linux, try MacOS
+        arch_dirs = glob(os.path.join(depends_dir, 'x86_64-apple-darwin*'))
+        if arch_dirs:
+            # Just try the first one; there will only be on in CI
+            arch_dir = arch_dirs[0]
+
+    exit_code = 0
+
+    if os.path.isdir(arch_dir):
+        lib_dir = os.path.join(arch_dir, 'lib')
+        libraries = os.listdir(lib_dir)
+
+        for lib in libraries:
+            if lib.find(".so") != -1:
+                print lib
+                exit_code = 1
+    else:
+        exit_code = 2
+        print "arch-specific build dir not present"
+        print "Did you build the ./depends tree?"
+        print "Are you on a currently unsupported architecture?"
+
+    if exit_code == 0:
+        print "PASS."
+    else:
+        print "FAIL."
+
+    return exit_code == 0
+
+def util_test():
+    return subprocess.call(
+        [repofile('src/test/bitcoin-util-test.py')],
+        cwd=repofile('src'),
+        env={'PYTHONPATH': repofile('src/test'), 'srcdir': repofile('src')}
+    ) == 0
+
+
+#
+# Tests
+#
+
+STAGES = [
+    'btest',
+    'gtest',
+    'sec-hard',
+    'no-dot-so',
+    'util-test',
+    'secp256k1',
+    'libsnark',
+    'univalue',
+    'rpc',
+]
+
+STAGE_COMMANDS = {
+    'btest': [repofile('src/test/test_bitcoin'), '-p'],
+    'gtest': [repofile('src/zcash-gtest')],
+    'sec-hard': check_security_hardening,
+    'no-dot-so': ensure_no_dot_so_in_depends,
+    'util-test': util_test,
+    'secp256k1': ['make', '-C', repofile('src/secp256k1'), 'check'],
+    'libsnark': ['make', '-C', repofile('src'), 'libsnark-tests'],
+    'univalue': ['make', '-C', repofile('src/univalue'), 'check'],
+    'rpc': [repofile('qa/pull-tester/rpc-tests.sh')],
+}
+
+
+#
+# Test driver
+#
+
+def run_stage(stage):
+    print('Running stage %s' % stage)
+    print('=' * (len(stage) + 14))
+    print
+
+    cmd = STAGE_COMMANDS[stage]
+    if type(cmd) == type([]):
+        ret = subprocess.call(cmd) == 0
+    else:
+        ret = cmd()
+
+    print
+    print('-' * (len(stage) + 15))
+    print('Finished stage %s' % stage)
+    print
+
+    return ret
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list-stages', dest='list', action='store_true')
+    parser.add_argument('stage', nargs='*', default=STAGES,
+                        help='One of %s'%STAGES)
+    args = parser.parse_args()
+
+    # Check for list
+    if args.list:
+        for s in STAGES:
+            print(s)
+        sys.exit(0)
+
+    # Check validity of stages
+    for s in args.stage:
+        if s not in STAGES:
+            print("Invalid stage '%s' (choose from %s)" % (s, STAGES))
+            sys.exit(1)
+
+    # Run the stages
+    passed = True
+    for s in args.stage:
+        passed &= run_stage(s)
+
+    if not passed:
+        print("!!! One or more test stages failed !!!")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

qa/hush/performance-measurements.sh

@@ -0,0 +1,341 @@
+#!/bin/bash
+set -u
+
+
+DATADIR=./benchmark-datadir
+SHA256CMD="$(command -v sha256sum || echo shasum)"
+SHA256ARGS="$(command -v sha256sum >/dev/null || echo '-a 256')"
+
+function zcash_rpc {
+    ./src/zcash-cli -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 "$@"
+}
+
+function zcash_rpc_slow {
+    # Timeout of 1 hour
+    zcash_rpc -rpcclienttimeout=3600 "$@"
+}
+
+function zcash_rpc_veryslow {
+    # Timeout of 2.5 hours
+    zcash_rpc -rpcclienttimeout=9000 "$@"
+}
+
+function zcash_rpc_wait_for_start {
+    zcash_rpc -rpcwait getinfo > /dev/null
+}
+
+function zcashd_generate {
+    zcash_rpc generate 101 > /dev/null
+}
+
+function extract_benchmark_datadir {
+    if [ -f "$1.tar.xz" ]; then
+        # Check the hash of the archive:
+        "$SHA256CMD" $SHA256ARGS -c <<EOF
+$2  $1.tar.xz
+EOF
+        ARCHIVE_RESULT=$?
+    else
+        echo "$1.tar.xz not found."
+        ARCHIVE_RESULT=1
+    fi
+    if [ $ARCHIVE_RESULT -ne 0 ]; then
+        zcashd_stop
+        echo
+        echo "Please download it and place it in the base directory of the repository."
+        exit 1
+    fi
+    xzcat "$1.tar.xz" | tar x
+}
+
+function use_200k_benchmark {
+    rm -rf benchmark-200k-UTXOs
+    extract_benchmark_datadir benchmark-200k-UTXOs dc8ab89eaa13730da57d9ac373c1f4e818a37181c1443f61fd11327e49fbcc5e
+    DATADIR="./benchmark-200k-UTXOs/node$1"
+}
+
+function zcashd_start {
+    case "$1" in
+        sendtoaddress|loadwallet|listunspent)
+            case "$2" in
+                200k-recv)
+                    use_200k_benchmark 0
+                    ;;
+                200k-send)
+                    use_200k_benchmark 1
+                    ;;
+                *)
+                    echo "Bad arguments to zcashd_start."
+                    exit 1
+            esac
+            ;;
+        *)
+            rm -rf "$DATADIR"
+            mkdir -p "$DATADIR/regtest"
+            touch "$DATADIR/zcash.conf"
+    esac
+    ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
+    ZCASHD_PID=$!
+    zcash_rpc_wait_for_start
+}
+
+function zcashd_stop {
+    zcash_rpc stop > /dev/null
+    wait $ZCASHD_PID
+}
+
+function zcashd_massif_start {
+    case "$1" in
+        sendtoaddress|loadwallet|listunspent)
+            case "$2" in
+                200k-recv)
+                    use_200k_benchmark 0
+                    ;;
+                200k-send)
+                    use_200k_benchmark 1
+                    ;;
+                *)
+                    echo "Bad arguments to zcashd_massif_start."
+                    exit 1
+            esac
+            ;;
+        *)
+            rm -rf "$DATADIR"
+            mkdir -p "$DATADIR/regtest"
+            touch "$DATADIR/zcash.conf"
+    esac
+    rm -f massif.out
+    valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
+    ZCASHD_PID=$!
+    zcash_rpc_wait_for_start
+}
+
+function zcashd_massif_stop {
+    zcash_rpc stop > /dev/null
+    wait $ZCASHD_PID
+    ms_print massif.out
+}
+
+function zcashd_valgrind_start {
+    rm -rf "$DATADIR"
+    mkdir -p "$DATADIR/regtest"
+    touch "$DATADIR/zcash.conf"
+    rm -f valgrind.out
+    valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
+    ZCASHD_PID=$!
+    zcash_rpc_wait_for_start
+}
+
+function zcashd_valgrind_stop {
+    zcash_rpc stop > /dev/null
+    wait $ZCASHD_PID
+    cat valgrind.out
+}
+
+function extract_benchmark_data {
+    if [ -f "block-107134.tar.xz" ]; then
+        # Check the hash of the archive:
+        "$SHA256CMD" $SHA256ARGS -c <<EOF
+4bd5ad1149714394e8895fa536725ed5d6c32c99812b962bfa73f03b5ffad4bb  block-107134.tar.xz
+EOF
+        ARCHIVE_RESULT=$?
+    else
+        echo "block-107134.tar.xz not found."
+        ARCHIVE_RESULT=1
+    fi
+    if [ $ARCHIVE_RESULT -ne 0 ]; then
+        zcashd_stop
+        echo
+        echo "Please generate it using qa/zcash/create_benchmark_archive.py"
+        echo "and place it in the base directory of the repository."
+        echo "Usage details are inside the Python script."
+        exit 1
+    fi
+    xzcat block-107134.tar.xz | tar x -C "$DATADIR/regtest"
+}
+
+
+if [ $# -lt 2 ]
+then
+    echo "$0 : At least two arguments are required!"
+    exit 1
+fi
+
+# Precomputation
+case "$1" in
+    *)
+        case "$2" in
+            verifyjoinsplit)
+                zcashd_start "${@:2}"
+                RAWJOINSPLIT=$(zcash_rpc zcsamplejoinsplit)
+                zcashd_stop
+        esac
+esac
+
+case "$1" in
+    time)
+        zcashd_start "${@:2}"
+        case "$2" in
+            sleep)
+                zcash_rpc zcbenchmark sleep 10
+                ;;
+            parameterloading)
+                zcash_rpc zcbenchmark parameterloading 10
+                ;;
+            createjoinsplit)
+                zcash_rpc zcbenchmark createjoinsplit 10 "${@:3}"
+                ;;
+            verifyjoinsplit)
+                zcash_rpc zcbenchmark verifyjoinsplit 1000 "\"$RAWJOINSPLIT\""
+                ;;
+            solveequihash)
+                zcash_rpc_slow zcbenchmark solveequihash 50 "${@:3}"
+                ;;
+            verifyequihash)
+                zcash_rpc zcbenchmark verifyequihash 1000
+                ;;
+            validatelargetx)
+                zcash_rpc zcbenchmark validatelargetx 10 "${@:3}"
+                ;;
+            trydecryptnotes)
+                zcash_rpc zcbenchmark trydecryptnotes 1000 "${@:3}"
+                ;;
+            incnotewitnesses)
+                zcash_rpc zcbenchmark incnotewitnesses 100 "${@:3}"
+                ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 10
+                ;;
+            sendtoaddress)
+                zcash_rpc zcbenchmark sendtoaddress 10 "${@:4}"
+                ;;
+            loadwallet)
+                zcash_rpc zcbenchmark loadwallet 10 
+                ;;
+            listunspent)
+                zcash_rpc zcbenchmark listunspent 10
+                ;;
+            *)
+                zcashd_stop
+                echo "Bad arguments to time."
+                exit 1
+        esac
+        zcashd_stop
+        ;;
+    memory)
+        zcashd_massif_start "${@:2}"
+        case "$2" in
+            sleep)
+                zcash_rpc zcbenchmark sleep 1
+                ;;
+            parameterloading)
+                zcash_rpc zcbenchmark parameterloading 1
+                ;;
+            createjoinsplit)
+                zcash_rpc_slow zcbenchmark createjoinsplit 1 "${@:3}"
+                ;;
+            verifyjoinsplit)
+                zcash_rpc zcbenchmark verifyjoinsplit 1 "\"$RAWJOINSPLIT\""
+                ;;
+            solveequihash)
+                zcash_rpc_slow zcbenchmark solveequihash 1 "${@:3}"
+                ;;
+            verifyequihash)
+                zcash_rpc zcbenchmark verifyequihash 1
+                ;;
+            validatelargetx)
+                zcash_rpc zcbenchmark validatelargetx 1
+                ;;
+            trydecryptnotes)
+                zcash_rpc zcbenchmark trydecryptnotes 1 "${@:3}"
+                ;;
+            incnotewitnesses)
+                zcash_rpc zcbenchmark incnotewitnesses 1 "${@:3}"
+                ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 1
+                ;;
+            sendtoaddress)
+                zcash_rpc zcbenchmark sendtoaddress 1 "${@:4}"
+                ;;
+            loadwallet)
+                # The initial load is sufficient for measurement
+                ;;
+            listunspent)
+                zcash_rpc zcbenchmark listunspent 1
+                ;;
+            *)
+                zcashd_massif_stop
+                echo "Bad arguments to memory."
+                exit 1
+        esac
+        zcashd_massif_stop
+        rm -f massif.out
+        ;;
+    valgrind)
+        zcashd_valgrind_start
+        case "$2" in
+            sleep)
+                zcash_rpc zcbenchmark sleep 1
+                ;;
+            parameterloading)
+                zcash_rpc zcbenchmark parameterloading 1
+                ;;
+            createjoinsplit)
+                zcash_rpc_veryslow zcbenchmark createjoinsplit 1 "${@:3}"
+                ;;
+            verifyjoinsplit)
+                zcash_rpc zcbenchmark verifyjoinsplit 1 "\"$RAWJOINSPLIT\""
+                ;;
+            solveequihash)
+                zcash_rpc_veryslow zcbenchmark solveequihash 1 "${@:3}"
+                ;;
+            verifyequihash)
+                zcash_rpc zcbenchmark verifyequihash 1
+                ;;
+            trydecryptnotes)
+                zcash_rpc zcbenchmark trydecryptnotes 1 "${@:3}"
+                ;;
+            incnotewitnesses)
+                zcash_rpc zcbenchmark incnotewitnesses 1 "${@:3}"
+                ;;
+            connectblockslow)
+                extract_benchmark_data
+                zcash_rpc zcbenchmark connectblockslow 1
+                ;;
+            *)
+                zcashd_valgrind_stop
+                echo "Bad arguments to valgrind."
+                exit 1
+        esac
+        zcashd_valgrind_stop
+        rm -f valgrind.out
+        ;;
+    valgrind-tests)
+        case "$2" in
+            gtest)
+                rm -f valgrind.out
+                valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/zcash-gtest
+                cat valgrind.out
+                rm -f valgrind.out
+                ;;
+            test_bitcoin)
+                rm -f valgrind.out
+                valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/test/test_bitcoin
+                cat valgrind.out
+                rm -f valgrind.out
+                ;;
+            *)
+                echo "Bad arguments to valgrind-tests."
+                exit 1
+        esac
+        ;;
+    *)
+        echo "Invalid benchmark type."
+        exit 1
+esac
+
+# Cleanup
+rm -rf "$DATADIR"

qa/hush/test-depends-sources-mirror.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python2
+
+# This script tests that the package mirror at https://z.cash/depends-sources/
+# contains all of the packages required to build this version of Zcash.
+#
+# This script assumes you've just built Zcash, and that as a result of that
+# build, all of the dependency packages have been downloaded into the
+# depends/sources directory (inside the root of this repository). The script
+# checks that all of those files are accessible on the mirror.
+
+import sys
+import os
+import requests
+
+MIRROR_URL_DIR="https://z.cash/depends-sources/"
+DEPENDS_SOURCES_DIR=os.path.realpath(os.path.join(
+    os.path.dirname(__file__),
+    "..", "..", "depends", "sources"
+))
+
+def get_depends_sources_list():
+    return filter(
+	lambda f: os.path.isfile(os.path.join(DEPENDS_SOURCES_DIR, f)),
+	os.listdir(DEPENDS_SOURCES_DIR)
+    )
+
+for filename in get_depends_sources_list():
+    resp = requests.head(MIRROR_URL_DIR + filename)
+
+    print "Checking [" + filename + "] ..."
+
+    if resp.status_code != 200:
+	print "FAIL. File not found on server: " + filename
+	sys.exit(1)
+
+    expected_size = os.path.getsize(os.path.join(DEPENDS_SOURCES_DIR, filename))
+    server_size = int(resp.headers['Content-Length'])
+    if expected_size != server_size:
+	print "FAIL. On the server, %s is %d bytes, but locally it is %d bytes." % (filename, server_size, expected_size)
+	sys.exit(1)
+
+print "PASS."
+sys.exit(0)