Ci update (#123)

2018-09-03 19:14:48 -07:00
parent 09345b2f0f
commit b748ba7fbf
2 changed files with 233 additions and 34 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,10 +5,10 @@ stages:

 variables:
  VERSION: 0.3.13
-  VERUS_CLI_LINUX: "verus-cli-linux-v${VERSION}-beta.tar.gz"
-  VERUS_CLI_WINDOWS: "verus-cli-windows-v${VERSION}-beta.zip"
-  VERUS_CLI_MACOS: "verus-cli-mac-v${VERSION}-beta.tar.gz"
-  POST_COMMENT: "Branch: ${CI_COMMIT_REF_NAME} Commit: ${CI_COMMIT_SHA}"
+  VERUS_CLI_LINUX: "Verus-CLI-Linux-v${VERSION}-beta.tar.gz"
+  VERUS_CLI_WINDOWS: "Verus-CLI-Windows-v${VERSION}-beta.zip"
+  VERUS_CLI_MACOS: "Verus-CLI-MacOS-v${VERSION}-beta.tar.gz"
+  POST_COMMENT: "Branch and Commit: ${CI_COMMIT_REF_NAME} ${CI_COMMIT_SHA} $'\n'MD5: "

 build:linux:
  image: asherd/veruscoin-cross-compiler:linux
@@ -22,12 +22,14 @@ build:linux:
  - zcutil/build.sh -j4
  - cp src/komodod src/komodo-cli kmd/linux/verus-cli
  - chmod +x kmd/linux/verus-cli/komodod
-      kmd/linux/verus-cli/komodo-cli
-      kmd/linux/verus-cli/verus
-      kmd/linux/verus-cli/verusd
-      kmd/linux/verus-cli/fetch-params
+  - chmod +x kmd/linux/verus-cli/komodo-cli
+  - chmod +x kmd/linux/verus-cli/verus
+  - chmod +x kmd/linux/verus-cli/verusd
+  - chmod +x kmd/linux/verus-cli/fetch-params
  - dos2unix kmd/linux/verus-cli/README.txt
-  - tar -C kmd/linux/ -czvf $VERUS_CLI_LINUX verus-cli ./
+  - cd kmd/linux/
+  - tar -czvf $VERUS_CLI_LINUX verus-cli
+  - mv $VERUS_CLI_LINUX ../..
  after_script:
  - mv /root/.ccache ./ || true
  cache:
@@ -78,8 +80,8 @@ build:mac:
    - depends/built
  script:
  - brew bundle
-  - zcutil/build-mac.sh | xcpretty
-  - ./makeReleaseMac.sh -j6
+  - zcutil/build-mac.sh -j6 | xcpretty
+  - ./makeReleaseMac.sh
  - dos2unix kmd/mac/verus-cli/README.txt
  - tar -C kmd/mac/ -czvf $VERUS_CLI_MACOS verus-cli ./
  artifacts:
@@ -87,7 +89,7 @@ build:mac:
    expire_in: 1 week


-code_quality:
+.code_quality:
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
@@ -105,7 +107,7 @@ code_quality:
    paths: [gl-code-quality-report.json]


-sast:
+.sast:
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
@@ -124,7 +126,7 @@ sast:



-license_management:
+.license_management:
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
@@ -140,7 +142,7 @@ license_management:
    paths: [gl-license-management-report.json]


-ubuntu:xenial:
+.ubuntu:xenial:
  image: ubuntu:xenial
  variables:
    DOCKER_DRIVER: overlay2
@@ -165,7 +167,7 @@ ubuntu:xenial:
  - build:linux


-ubuntu:bionic:
+.ubuntu:bionic:
  image: ubuntu:bionic
  variables:
    DOCKER_DRIVER: overlay2
@@ -241,36 +243,41 @@ deploy:
  - build:linux
  - build:windows
  - build:mac
+  before_script:
+  - export VERUS_CLI_LINUX_MD5=$(md5sum $VERUS_CLI_LINUX | cut -d " " -f 1)
+  - export VERUS_CLI_WINDOWS_MD5=$(md5sum $VERUS_CLI_WINDOWS | cut -d " " -f 1)
+  - export VERUS_CLI_MACOS_MD5=$(md5sum $VERUS_CLI_MACOS | cut -d " " -f 1)
  script:
+  - export VERUS_CLI_LINUX_MD5=$(md5sum $VERUS_CLI_LINUX | cut -d " " -f 1)
  - curl -F file=@"$VERUS_CLI_LINUX"
      -F channels="$CLI_POST_CHANNEL"
-      -F initial_comment="${POST_COMMENT}"
+      -F initial_comment="${POST_COMMENT}$VERUS_CLI_LINUX_MD5"
      -H "${SLACK_BOT_AUTH}"
      "https://slack.com/api/files.upload"
  - curl -F file=@"$VERUS_CLI_WINDOWS"
      -F channels="$CLI_POST_CHANNEL"
-      -F initial_comment="${POST_COMMENT}"
+      -F initial_comment="${POST_COMMENT}$VERUS_CLI_WINDOWS_MD5"
      -H "${SLACK_BOT_AUTH}"
      "https://slack.com/api/files.upload"
  - curl -F file=@"$VERUS_CLI_MACOS"
      -F channels="$CLI_POST_CHANNEL"
-      -F initial_comment="${POST_COMMENT}"
+      -F initial_comment="${POST_COMMENT}$VERUS_CLI_MACOS_MD5"
      -H "${SLACK_BOT_AUTH}"
      "https://slack.com/api/files.upload"
-  - mkdir Windows
-  - mkdir Linux
-  - mkdir MacOS
-  - mv $VERUS_CLI_WINDOWS Windows
-  - mv $VERUS_CLI_LINUX Linux
-  - mv $VERUS_CLI_MACOS MacOS
-  - echo "$AUTH_KEY" > AUTH_KEY.json
-  - gcloud auth activate-service-account
-      --key-file AUTH_KEY.json
+  - mkdir Windows && mkdir Linux && mkdir MacOS &&
+    mv $VERUS_CLI_WINDOWS Windows &&
+    mv $VERUS_CLI_LINUX Linux &&
+    mv $VERUS_CLI_MACOS MacOS
+  - echo "$AUTH_KEY" > AUTH_KEY.json &&
+    gcloud auth activate-service-account
+    --key-file AUTH_KEY.json
  - gsutil cp -r Windows Linux MacOS $STAGING/VerusCoin/$CI_COMMIT_REF_NAME/
  - curl -X POST
      -F token="$CI_JOB_TOKEN"
-      -F ref=master
-      -F variables\[CLI_VERSION\]="$VERSION"
-      -F variables\[AGAMA_VERSION\]="$VERSION"
-      "https://gitlab.com/api/v4/projects/8018638/trigger/pipeline"
+      -F ref=dev
+      -F variables\[UPSTREAM_CLI_BRANCH\]="$CI_COMMIT_REF_NAME"
+      -F variables\[VERUS_CLI_LINUX\]="$VERUS_CLI_LINUX"
+      -F variables\[VERUS_CLI_WINDOWS\]="$VERUS_CLI_WINDOWS"
+      -F variables\[VERUS_CLI_MACOS\]="$VERUS_CLI_MACOS"
+      "https://gitlab.com/api/v4/projects/8018592/trigger/pipeline"

--- a/kmd/mac/verus-cli/fetch-params
+++ b/kmd/mac/verus-cli/fetch-params
@@ -1,7 +1,199 @@
 #!/bin/bash

 set -eu
-curl "https://z.cash/downloads/sprout-proving.key" -o "$HOME/Library/Application Support/ZcashParams/sprout-proving.key"
-curl "https://z.cash/downloads/sprout-verifying.key" -o "$HOME/Library/Application Support/ZcashParams/sprout-verifying.key"

+if [[ "$OSTYPE" == "darwin"* ]]; then
+    PARAMS_DIR="$HOME/Library/Application Support/ZcashParams"
+else
+    PARAMS_DIR="$HOME/.zcash-params"
+fi

+SPROUT_PKEY_NAME='sprout-proving.key'
+SPROUT_VKEY_NAME='sprout-verifying.key'
+SAPLING_SPEND_NAME='sapling-spend.params'
+SAPLING_OUTPUT_NAME='sapling-output.params'
+SAPLING_SPROUT_GROTH16_NAME='sprout-groth16.params'
+SPROUT_URL="https://z.cash/downloads"
+SPROUT_IPFS="/ipfs/QmZKKx7Xup7LiAtFRhYsE1M7waXcv9ir9eCECyXAFGxhEo"
+
+SHA256CMD="$(command -v sha256sum || echo shasum)"
+SHA256ARGS="$(command -v sha256sum >/dev/null || echo '-a 256')"
+
+WGETCMD="$(command -v wget || echo '')"
+IPFSCMD="$(command -v ipfs || echo '')"
+CURLCMD="$(command -v curl || echo '')"
+
+# fetch methods can be disabled with ZC_DISABLE_SOMETHING=1
+ZC_DISABLE_WGET="${ZC_DISABLE_WGET:-}"
+ZC_DISABLE_IPFS="${ZC_DISABLE_IPFS:-}"
+ZC_DISABLE_CURL="${ZC_DISABLE_CURL:-}"
+
+function fetch_wget {
+    if [ -z "$WGETCMD" ] || ! [ -z "$ZC_DISABLE_WGET" ]; then
+        return 1
+    fi
+
+    local filename="$1"
+    local dlname="$2"
+
+    cat <<EOF
+Retrieving (wget): $SPROUT_URL/$filename
+EOF
+
+    wget \
+        --progress=dot:giga \
+        --output-document="$dlname" \
+        --continue \
+        --retry-connrefused --waitretry=3 --timeout=30 \
+        "$SPROUT_URL/$filename"
+}
+
+function fetch_ipfs {
+    if [ -z "$IPFSCMD" ] || ! [ -z "$ZC_DISABLE_IPFS" ]; then
+        return 1
+    fi
+
+    local filename="$1"
+    local dlname="$2"
+
+    cat <<EOF
+Retrieving (ipfs): $SPROUT_IPFS/$filename
+EOF
+
+    ipfs get --output "$dlname" "$SPROUT_IPFS/$filename"
+}
+
+function fetch_curl {
+    if [ -z "$CURLCMD" ] || ! [ -z "$ZC_DISABLE_CURL" ]; then
+        return 1
+    fi
+
+    local filename="$1"
+    local dlname="$2"
+
+    cat <<EOF
+Retrieving (curl): $SPROUT_URL/$filename
+EOF
+
+    curl \
+        --output "$dlname" \
+        -# -L -C - \
+        "$SPROUT_URL/$filename"
+
+}
+
+function fetch_failure {
+    cat >&2 <<EOF
+Failed to fetch the Zcash zkSNARK parameters!
+Try installing one of the following programs and make sure you're online:
+ * ipfs
+ * wget
+ * curl
+EOF
+    exit 1
+}
+
+function fetch_params {
+    local filename="$1"
+    local output="$2"
+    local dlname="${output}.dl"
+    local expectedhash="$3"
+
+    if ! [ -f "$output" ]
+    then
+        for method in wget ipfs curl failure; do
+            if "fetch_$method" "$filename" "$dlname"; then
+                echo "Download successful!"
+                break
+            fi
+        done
+
+        "$SHA256CMD" $SHA256ARGS -c <<EOF
+$expectedhash  $dlname
+EOF
+
+        # Check the exit code of the shasum command:
+        CHECKSUM_RESULT=$?
+        if [ $CHECKSUM_RESULT -eq 0 ]; then
+            mv -v "$dlname" "$output"
+        else
+            echo "Failed to verify parameter checksums!" >&2
+            exit 1
+        fi
+    fi
+}
+
+# Use flock to prevent parallel execution.
+function lock() {
+    local lockfile=/tmp/fetch_params.lock
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        if shlock -f ${lockfile} -p $$; then
+            return 0
+        else
+            return 1
+        fi
+    else
+        # create lock file
+        eval "exec 200>$lockfile"
+        # acquire the lock
+        flock -n 200 \
+            && return 0 \
+            || return 1
+    fi
+}
+
+function exit_locked_error {
+    echo "Only one instance of fetch-params.sh can be run at a time." >&2
+    exit 1
+}
+
+function main() {
+
+    lock fetch-params.sh \
+    || exit_locked_error
+
+    cat <<EOF
+Zcash - fetch-params.sh
+This script will fetch the Zcash zkSNARK parameters and verify their
+integrity with sha256sum.
+If they already exist locally, it will exit now and do nothing else.
+EOF
+
+    # Now create PARAMS_DIR and insert a README if necessary:
+    if ! [ -d "$PARAMS_DIR" ]
+    then
+        mkdir -p "$PARAMS_DIR"
+        README_PATH="$PARAMS_DIR/README"
+        cat >> "$README_PATH" <<EOF
+This directory stores common Zcash zkSNARK parameters. Note that it is
+distinct from the daemon's -datadir argument because the parameters are
+large and may be shared across multiple distinct -datadir's such as when
+setting up test networks.
+EOF
+
+        # This may be the first time the user's run this script, so give
+        # them some info, especially about bandwidth usage:
+        cat <<EOF
+The parameters are currently just under 911MB in size, so plan accordingly
+for your bandwidth constraints. If the files are already present and
+have the correct sha256sum, no networking is used.
+Creating params directory. For details about this directory, see:
+$README_PATH
+EOF
+    fi
+
+    cd "$PARAMS_DIR"
+
+    # Sprout parameters:
+    fetch_params "$SPROUT_PKEY_NAME" "$PARAMS_DIR/$SPROUT_PKEY_NAME" "8bc20a7f013b2b58970cddd2e7ea028975c88ae7ceb9259a5344a16bc2c0eef7"
+    fetch_params "$SPROUT_VKEY_NAME" "$PARAMS_DIR/$SPROUT_VKEY_NAME" "4bd498dae0aacfd8e98dc306338d017d9c08dd0918ead18172bd0aec2fc5df82"
+
+    # Sapling parameters:
+    fetch_params "$SAPLING_SPEND_NAME" "$PARAMS_DIR/$SAPLING_SPEND_NAME" "8e48ffd23abb3a5fd9c5589204f32d9c31285a04b78096ba40a79b75677efc13"
+    fetch_params "$SAPLING_OUTPUT_NAME" "$PARAMS_DIR/$SAPLING_OUTPUT_NAME" "2f0ebbcbb9bb0bcffe95a397e7eba89c29eb4dde6191c339db88570e3f3fb0e4"
+    fetch_params "$SAPLING_SPROUT_GROTH16_NAME" "$PARAMS_DIR/$SAPLING_SPROUT_GROTH16_NAME" "b685d700c60328498fbde589c8c7c484c722b788b265b72af448a5bf0ee55b50"
+}
+
+main
+rm -f /tmp/fetch_params.lock
+exit 0