Merge remote-tracking branch 'jl777/FSM' into duke

depends/packages/libcurl.mk

@@ -1,9 +1,10 @@
 package=libcurl
-$(package)_version=7.54.0
+$(package)_version=7.64.1
+$(package)_dependencies=openssl
 $(package)_download_path=https://curl.haxx.se/download
 $(package)_file_name=curl-$($(package)_version).tar.gz
-$(package)_sha256_hash=a84b635941c74e26cce69dd817489bec687eb1f230e7d1897fc5b5f108b59adf
+$(package)_sha256_hash=432d3f466644b9416bc5b649d344116a753aeaa520c8beaf024a90cba9d3d35d
-$(package)_config_opts_linux=--disable-shared --enable-static --prefix=$(host_prefix)
+$(package)_config_opts_linux=--disable-shared --enable-static --prefix=$(host_prefix) --host=x86_64-unknown-linux-gnu
 $(package)_config_opts_mingw32=--enable-mingw --disable-shared --enable-static --prefix=$(host_prefix) --host=x86_64-w64-mingw32
 $(package)_config_opts_darwin=--disable-shared --enable-static --prefix=$(host_prefix)
 $(package)_cflags_darwin=-mmacosx-version-min=10.9
@@ -15,11 +16,20 @@ define $(package)_set_vars
 endef
 endif
 
-define $(package)_config_cmds
+ifeq ($(build_os),linux)
-  $($(package)_conf_tool) $($(package)_config_opts)
+define $(package)_set_vars
+  $(package)_config_env=LD_LIBRARY_PATH="$(host_prefix)/lib" PKG_CONFIG_LIBDIR="$(host_prefix)/lib/pkgconfig" CPPFLAGS="-I$(host_prefix)/include" LDFLAGS="-L$(host_prefix)/lib"
 endef
+endif
 
 
+define $(package)_config_cmds
+  echo '=== config for $(package):' && \
+  echo '$($(package)_config_env) $($(package)_conf_tool) $($(package)_config_opts)' && \
+  echo '=== ' && \
+  $($(package)_config_env) $($(package)_conf_tool) $($(package)_config_opts) 
+endef
+
 ifeq ($(build_os),darwin)
 define $(package)_build_cmds
   $(MAKE) CPPFLAGS='-fPIC' CFLAGS='-mmacosx-version-min=10.9'
@@ -31,5 +41,6 @@ endef
 endif
 
 define $(package)_stage_cmds
+  echo 'Staging dir: $($(package)_staging_dir)$(host_prefix)/' && \
   $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef

depends/packages/openssl.mk

@@ -3,6 +3,7 @@ $(package)_version=1.1.1a
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+$(package)_patches=ssl_fix.patch
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
@@ -22,7 +23,6 @@ $(package)_config_opts+=no-comp
 $(package)_config_opts+=no-crypto-mdebug
 $(package)_config_opts+=no-crypto-mdebug-backtrace
 $(package)_config_opts+=no-ct
-$(package)_config_opts+=no-des
 $(package)_config_opts+=no-dgram
 $(package)_config_opts+=no-dsa
 $(package)_config_opts+=no-dso
@@ -58,15 +58,12 @@ $(package)_config_opts+=no-scrypt
 $(package)_config_opts+=no-sctp
 $(package)_config_opts+=no-seed
 $(package)_config_opts+=no-shared
-$(package)_config_opts+=no-sock
 $(package)_config_opts+=no-srp
 $(package)_config_opts+=no-srtp
-$(package)_config_opts+=no-ssl
 $(package)_config_opts+=no-ssl3
 $(package)_config_opts+=no-ssl3-method
 $(package)_config_opts+=no-ssl-trace
 $(package)_config_opts+=no-stdio
-$(package)_config_opts+=no-tls
 $(package)_config_opts+=no-tls1
 $(package)_config_opts+=no-tls1-method
 $(package)_config_opts+=no-ts
@@ -93,7 +90,8 @@ endef
 
 define $(package)_preprocess_cmds
   sed -i.old 's/built on: $date/built on: not available/' util/mkbuildinf.pl && \
-  sed -i.old "s|\"engines\", \"apps\", \"test\"|\"engines\"|" Configure
+  sed -i.old "s|\"engines\", \"apps\", \"test\"|\"engines\"|" Configure && \
+  patch -p1 < $($(package)_patch_dir)/ssl_fix.patch
 endef
 
 define $(package)_config_cmds

depends/packages/packages.mk

@@ -47,7 +47,7 @@ native_packages := native_ccache
 wallet_packages=bdb
 
 ifeq ($(host_os),linux)
-	packages := boost openssl libevent zeromq $(zcash_packages) googletest #googlemock
+	packages := boost openssl libevent zeromq $(zcash_packages) googletest libcurl #googlemock
 else
 	packages := boost openssl libevent zeromq $(zcash_packages) libcurl googletest #googlemock
 endif

depends/patches/openssl/ssl_fix.patch

@@ -0,0 +1,273 @@
+From f725fe5b4b6504df08e30f5194d321c3025e2336 Mon Sep 17 00:00:00 2001
+From: Matt Caswell <matt@openssl.org>
+Date: Tue, 20 Nov 2018 15:32:55 +0000
+Subject: [PATCH] Fix a RUN_ONCE bug
+
+We have a number of instances where there are multiple "init" functions for
+a single CRYPTO_ONCE variable, e.g. to load config automatically or to not
+load config automatically. Unfortunately the RUN_ONCE mechanism was not
+correctly giving the right return value where an alternative init function
+was being used.
+
+Reviewed-by: Tim Hudson <tjh@openssl.org>
+(Merged from https://github.com/openssl/openssl/pull/7983)
+---
+ crypto/init.c                  | 38 +++++++++-----
+ include/internal/thread_once.h | 92 ++++++++++++++++++++++++++++++++++
+ ssl/ssl_init.c                 |  6 ++-
+ 3 files changed, 121 insertions(+), 15 deletions(-)
+
+diff --git a/crypto/init.c b/crypto/init.c
+index 209d1a483da..f20a12f069a 100644
+--- a/crypto/init.c
++++ b/crypto/init.c
+@@ -177,12 +177,6 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_nodelete)
+ 
+ static CRYPTO_ONCE load_crypto_strings = CRYPTO_ONCE_STATIC_INIT;
+ static int load_crypto_strings_inited = 0;
+-DEFINE_RUN_ONCE_STATIC(ossl_init_no_load_crypto_strings)
+-{
+-    /* Do nothing in this case */
+-    return 1;
+-}
+-
+ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_strings)
+ {
+     int ret = 1;
+@@ -201,6 +195,13 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_crypto_strings)
+     return ret;
+ }
+ 
++DEFINE_RUN_ONCE_STATIC_ALT(ossl_init_no_load_crypto_strings,
++                           ossl_init_load_crypto_strings)
++{
++    /* Do nothing in this case */
++    return 1;
++}
++
+ static CRYPTO_ONCE add_all_ciphers = CRYPTO_ONCE_STATIC_INIT;
+ DEFINE_RUN_ONCE_STATIC(ossl_init_add_all_ciphers)
+ {
+@@ -218,6 +219,13 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_add_all_ciphers)
+     return 1;
+ }
+ 
++DEFINE_RUN_ONCE_STATIC_ALT(ossl_init_no_add_all_ciphers,
++                           ossl_init_add_all_ciphers)
++{
++    /* Do nothing */
++    return 1;
++}
++
+ static CRYPTO_ONCE add_all_digests = CRYPTO_ONCE_STATIC_INIT;
+ DEFINE_RUN_ONCE_STATIC(ossl_init_add_all_digests)
+ {
+@@ -235,7 +243,8 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_add_all_digests)
+     return 1;
+ }
+ 
+-DEFINE_RUN_ONCE_STATIC(ossl_init_no_add_algs)
++DEFINE_RUN_ONCE_STATIC_ALT(ossl_init_no_add_all_digests,
++                           ossl_init_add_all_digests)
+ {
+     /* Do nothing */
+     return 1;
+@@ -255,7 +264,7 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_config)
+     config_inited = 1;
+     return 1;
+ }
+-DEFINE_RUN_ONCE_STATIC(ossl_init_no_config)
++DEFINE_RUN_ONCE_STATIC_ALT(ossl_init_no_config, ossl_init_config)
+ {
+ #ifdef OPENSSL_INIT_DEBUG
+     fprintf(stderr,
+@@ -595,8 +604,9 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings)
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_NO_LOAD_CRYPTO_STRINGS)
+-            && !RUN_ONCE(&load_crypto_strings,
+-                         ossl_init_no_load_crypto_strings))
++            && !RUN_ONCE_ALT(&load_crypto_strings,
++                             ossl_init_no_load_crypto_strings,
++                             ossl_init_load_crypto_strings))
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_LOAD_CRYPTO_STRINGS)
+@@ -604,7 +614,8 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings)
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_NO_ADD_ALL_CIPHERS)
+-            && !RUN_ONCE(&add_all_ciphers, ossl_init_no_add_algs))
++            && !RUN_ONCE_ALT(&add_all_ciphers, ossl_init_no_add_all_ciphers,
++                             ossl_init_add_all_ciphers))
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_ADD_ALL_CIPHERS)
+@@ -612,7 +623,8 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings)
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_NO_ADD_ALL_DIGESTS)
+-            && !RUN_ONCE(&add_all_digests, ossl_init_no_add_algs))
++            && !RUN_ONCE_ALT(&add_all_digests, ossl_init_no_add_all_digests,
++                             ossl_init_add_all_digests))
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_ADD_ALL_DIGESTS)
+@@ -624,7 +636,7 @@ int OPENSSL_init_crypto(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings)
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_NO_LOAD_CONFIG)
+-            && !RUN_ONCE(&config, ossl_init_no_config))
++            && !RUN_ONCE_ALT(&config, ossl_init_no_config, ossl_init_config))
+         return 0;
+ 
+     if (opts & OPENSSL_INIT_LOAD_CONFIG) {
+diff --git a/include/internal/thread_once.h b/include/internal/thread_once.h
+index 224244353ab..e268a959ef3 100644
+--- a/include/internal/thread_once.h
++++ b/include/internal/thread_once.h
+@@ -9,6 +9,20 @@
+ 
+ #include <openssl/crypto.h>
+ 
++/*
++ * DEFINE_RUN_ONCE: Define an initialiser function that should be run exactly
++ * once. It takes no arguments and returns and int result (1 for success or
++ * 0 for failure). Typical usage might be:
++ *
++ * DEFINE_RUN_ONCE(myinitfunc)
++ * {
++ *     do_some_initialisation();
++ *     if (init_is_successful())
++ *         return 1;
++ *
++ *     return 0;
++ * }
++ */
+ #define DEFINE_RUN_ONCE(init)                   \
+     static int init(void);                     \
+     int init##_ossl_ret_ = 0;                   \
+@@ -17,10 +31,30 @@
+         init##_ossl_ret_ = init();              \
+     }                                           \
+     static int init(void)
++
++/*
++ * DECLARE_RUN_ONCE: Declare an initialiser function that should be run exactly
++ * once that has been defined in another file via DEFINE_RUN_ONCE().
++ */
+ #define DECLARE_RUN_ONCE(init)                  \
+     extern int init##_ossl_ret_;                \
+     void init##_ossl_(void);
+ 
++/*
++ * DEFINE_RUN_ONCE_STATIC: Define an initialiser function that should be run
++ * exactly once. This function will be declared as static within the file. It
++ * takes no arguments and returns and int result (1 for success or 0 for
++ * failure). Typical usage might be:
++ *
++ * DEFINE_RUN_ONCE_STATIC(myinitfunc)
++ * {
++ *     do_some_initialisation();
++ *     if (init_is_successful())
++ *         return 1;
++ *
++ *     return 0;
++ * }
++ */
+ #define DEFINE_RUN_ONCE_STATIC(init)            \
+     static int init(void);                     \
+     static int init##_ossl_ret_ = 0;            \
+@@ -30,6 +64,46 @@
+     }                                           \
+     static int init(void)
+ 
++/*
++ * DEFINE_RUN_ONCE_STATIC_ALT: Define an alternative initialiser function. This
++ * function will be declared as static within the file. It takes no arguments
++ * and returns an int result (1 for success or 0 for failure). An alternative
++ * initialiser function is expected to be associated with a primary initialiser
++ * function defined via DEFINE_ONCE_STATIC where both functions use the same
++ * CRYPTO_ONCE object to synchronise. Where an alternative initialiser function
++ * is used only one of the primary or the alternative initialiser function will
++ * ever be called - and that function will be called exactly once. Definitition
++ * of an alternative initialiser function MUST occur AFTER the definition of the
++ * primary initialiser function.
++ *
++ * Typical usage might be:
++ *
++ * DEFINE_RUN_ONCE_STATIC(myinitfunc)
++ * {
++ *     do_some_initialisation();
++ *     if (init_is_successful())
++ *         return 1;
++ *
++ *     return 0;
++ * }
++ *
++ * DEFINE_RUN_ONCE_STATIC_ALT(myaltinitfunc, myinitfunc)
++ * {
++ *     do_some_alternative_initialisation();
++ *     if (init_is_successful())
++ *         return 1;
++ *
++ *     return 0;
++ * }
++ */
++#define DEFINE_RUN_ONCE_STATIC_ALT(initalt, init) \
++    static int initalt(void);                     \
++    static void initalt##_ossl_(void)             \
++    {                                             \
++        init##_ossl_ret_ = initalt();             \
++    }                                             \
++    static int initalt(void)
++
+ /*
+  * RUN_ONCE - use CRYPTO_THREAD_run_once, and check if the init succeeded
+  * @once: pointer to static object of type CRYPTO_ONCE
+@@ -43,3 +117,21 @@
+  */
+ #define RUN_ONCE(once, init)                                            \
+     (CRYPTO_THREAD_run_once(once, init##_ossl_) ? init##_ossl_ret_ : 0)
++
++/*
++ * RUN_ONCE_ALT - use CRYPTO_THREAD_run_once, to run an alternative initialiser
++ *                function and check if that initialisation succeeded
++ * @once:    pointer to static object of type CRYPTO_ONCE
++ * @initalt: alternative initialiser function name that was previously given to
++ *           DEFINE_RUN_ONCE_STATIC_ALT.  This function must return 1 for
++ *           success or 0 for failure.
++ * @init:    primary initialiser function name that was previously given to
++ *           DEFINE_RUN_ONCE_STATIC.  This function must return 1 for success or
++ *           0 for failure.
++ *
++ * The return value is 1 on success (*) or 0 in case of error.
++ *
++ * (*) by convention, since the init function must return 1 on success.
++ */
++#define RUN_ONCE_ALT(once, initalt, init)                               \
++    (CRYPTO_THREAD_run_once(once, initalt##_ossl_) ? init##_ossl_ret_ : 0)
+diff --git a/ssl/ssl_init.c b/ssl/ssl_init.c
+index c0ccb9304a6..96526472c57 100644
+--- a/ssl/ssl_init.c
++++ b/ssl/ssl_init.c
+@@ -134,7 +134,8 @@ DEFINE_RUN_ONCE_STATIC(ossl_init_load_ssl_strings)
+     return 1;
+ }
+ 
+-DEFINE_RUN_ONCE_STATIC(ossl_init_no_load_ssl_strings)
++DEFINE_RUN_ONCE_STATIC_ALT(ossl_init_no_load_ssl_strings,
++                           ossl_init_load_ssl_strings)
+ {
+     /* Do nothing in this case */
+     return 1;
+@@ -207,7 +208,8 @@ int OPENSSL_init_ssl(uint64_t opts, const OPENSSL_INIT_SETTINGS * settings)
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_NO_LOAD_SSL_STRINGS)
+-        && !RUN_ONCE(&ssl_strings, ossl_init_no_load_ssl_strings))
++        && !RUN_ONCE_ALT(&ssl_strings, ossl_init_no_load_ssl_strings,
++                         ossl_init_load_ssl_strings))
+         return 0;
+ 
+     if ((opts & OPENSSL_INIT_LOAD_SSL_STRINGS)

qa/pull-tester/rpc-tests.sh

@@ -65,6 +65,7 @@ testScripts=(
     'disablewallet.py'
     'zcjoinsplit.py'
     'zcjoinsplitdoublespend.py'
+    'ivk_import_export.py'
     'zkey_import_export.py'
     'reorg_limit.py'
     'getblocktemplate.py'

qa/rpc-tests/ivk_import_export.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python2
+# Copyright (c) 2019 Bartlomiej Lisiecki
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+from decimal import Decimal
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, assert_greater_than, start_nodes,\
+    initialize_chain_clean, connect_nodes_bi, wait_and_assert_operationid_status
+
+import logging
+
+logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
+fee = Decimal('0.0001') # constant (but can be changed within reason)
+
+class IVKImportExportTest (BitcoinTestFramework):
+
+    def setup_chain(self):
+        print("Initializing test directory "+self.options.tmpdir)
+        initialize_chain_clean(self.options.tmpdir, 4)
+
+    def setup_network(self, split=False):
+        self.nodes = start_nodes(4, self.options.tmpdir, [[
+            '-nuparams=5ba81b19:101', # Overwinter
+            '-nuparams=76b809bb:102', # Sapling
+        ]] * 4)
+
+        connect_nodes_bi(self.nodes,0,1)
+        connect_nodes_bi(self.nodes,0,2)
+        connect_nodes_bi(self.nodes,0,3)
+        self.is_network_split=False
+        self.sync_all()
+
+    def run_test(self):
+        [alice, bob, charlie, miner] = self.nodes
+
+        # the sender loses 'amount' plus fee; to_addr receives exactly 'amount'
+        def z_send(from_node, from_addr, to_addr, amount):
+            global fee
+            opid = from_node.z_sendmany(from_addr,
+                [{"address": to_addr, "amount": Decimal(amount)}], 1, fee)
+            wait_and_assert_operationid_status(from_node, opid)
+            self.sync_all()
+            miner.generate(1)
+            self.sync_all()
+
+        def verify_utxos(node, amts, zaddr):
+            amts.sort(reverse=True)
+            txs = node.z_listreceivedbyaddress(zaddr)
+
+            def cmp_confirmations_high_to_low(a, b):
+                return cmp(b["amount"], a["amount"])
+
+            txs.sort(cmp_confirmations_high_to_low)
+            print("Sorted txs", txs)
+            print("amts", amts)
+
+            try:
+                assert_equal(amts, [tx["amount"] for tx in txs])
+            except AssertionError:
+                logging.error(
+                    'Expected amounts: %r; txs: %r',
+                    amts, txs)
+                raise
+
+        def get_private_balance(node):
+            balance = node.z_gettotalbalance()
+            return balance['private']
+
+        def find_imported_zaddr(node, import_zaddr):
+            zaddrs = node.z_listaddresses()
+            assert(import_zaddr in zaddrs)
+            return import_zaddr
+
+        # activate sapling
+        alice.generate(102)
+        self.sync_all()
+
+        # sanity-check the test harness
+        assert_equal(self.nodes[0].getblockcount(), 102)
+
+        # shield alice's coinbase funds to her zaddr
+        alice_zaddr = alice.z_getnewaddress('sapling')
+        res = alice.z_shieldcoinbase("*", alice_zaddr)
+        wait_and_assert_operationid_status(alice, res['opid'])
+        self.sync_all()
+        miner.generate(1)
+        self.sync_all()
+
+        # the amounts of each txn embodied which generates a single utxo:
+        amounts = map(Decimal, ['2.3', '3.7', '0.1', '0.5', '1.0', '0.19'])
+
+        # internal test consistency assertion:
+        assert_greater_than(
+            get_private_balance(alice),
+            reduce(Decimal.__add__, amounts))
+
+
+        # now get a pristine z-address for receiving transfers:
+        bob_zaddr = bob.z_getnewaddress('sapling')
+        verify_utxos(bob, [], bob_zaddr)
+
+        logging.info("sending pre-export txns...")
+        for amount in amounts[0:2]:
+            z_send(alice, alice_zaddr, bob_zaddr, amount)
+
+        logging.info("exporting ivk from bob...")
+        bob_ivk = bob.z_exportviewingkey(bob_zaddr)
+
+        logging.info("sending post-export txns...")
+        for amount in amounts[2:4]:
+            z_send(alice, alice_zaddr, bob_zaddr, amount)
+
+        verify_utxos(bob, amounts[:4], bob_zaddr)
+
+        logging.info("importing bob_ivk into charlie...")
+        # we need to pass bob_zaddr since it's a sapling address
+        charlie.z_importviewingkey(bob_ivk, 'yes', 0, bob_zaddr)
+
+        # z_importkey should have rescanned for new key, so this should pass:
+        verify_utxos(charlie, amounts[:4], bob_zaddr)
+
+        # verify idempotent behavior:
+        charlie.z_importviewingkey(bob_ivk, 'yes', 0, bob_zaddr)
+        verify_utxos(charlie, amounts[:4], bob_zaddr)
+
+
+        logging.info("Sending post-import txns...")
+        for amount in amounts[4:]:
+            z_send(alice, alice_zaddr, bob_zaddr, amount)
+
+        verify_utxos(bob, amounts, bob_zaddr)
+        verify_utxos(charlie, amounts, bob_zaddr)
+
+if __name__ == '__main__':
+    IVKImportExportTest().main()

src/gtest/test_keys.cpp

@@ -44,5 +44,19 @@ TEST(Keys, EncodeAndDecodeSapling)
             auto addr2 = boost::get<libzcash::SaplingPaymentAddress>(paymentaddr2);
             EXPECT_EQ(addr, addr2);
         }
+        {
+            auto ivk = sk.ToXFVK().fvk.in_viewing_key();
+            std::string ivk_string = EncodeViewingKey(ivk);
+            EXPECT_EQ(
+                ivk_string.substr(0, 5),
+                Params().Bech32HRP(CChainParams::SAPLING_INCOMING_VIEWING_KEY));
+
+            auto viewing_key = DecodeViewingKey(ivk_string);
+            EXPECT_TRUE(IsValidViewingKey(viewing_key));
+
+            auto ivk2 = boost::get<libzcash::SaplingIncomingViewingKey>(&viewing_key);
+            ASSERT_TRUE(ivk2 != nullptr);
+            EXPECT_EQ(*ivk2, ivk);
+        }
     }
 }

src/key_io.cpp

@@ -128,6 +128,19 @@ public:
         return ret;
     }
 
+    std::string operator()(const libzcash::SaplingIncomingViewingKey& vk) const
+    {
+        CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
+        ss << vk;
+        std::vector<unsigned char> serkey(ss.begin(), ss.end());
+        std::vector<unsigned char> data;
+        ConvertBits<8, 5, true>([&](unsigned char c) { data.push_back(c); }, serkey.begin(), serkey.end());
+        std::string ret = bech32::Encode(m_params.Bech32HRP(CChainParams::SAPLING_INCOMING_VIEWING_KEY), data);
+        memory_cleanse(serkey.data(), serkey.size());
+        memory_cleanse(data.data(), data.size());
+        return ret;
+    }
+
     std::string operator()(const libzcash::InvalidEncoding& no) const { return {}; }
 };
 
@@ -175,6 +188,7 @@ public:
 // perform ceiling division to get the number of 5-bit clusters.
 const size_t ConvertedSaplingPaymentAddressSize = ((32 + 11) * 8 + 4) / 5;
 const size_t ConvertedSaplingExtendedSpendingKeySize = (ZIP32_XSK_SIZE * 8 + 4) / 5;
+const size_t ConvertedSaplingIncomingViewingKeySize = (32 * 8 + 4) / 5;
 } // namespace
 
 CKey DecodeSecret(const std::string& str)
@@ -346,7 +360,19 @@ libzcash::ViewingKey DecodeViewingKey(const std::string& str)
             return ret;
         }
     }
-    memory_cleanse(data.data(), data.size());
+    data.clear();
+    auto bech = bech32::Decode(str);
+    if(bech.first == Params().Bech32HRP(CChainParams::SAPLING_INCOMING_VIEWING_KEY) &&
+       bech.second.size() == ConvertedSaplingIncomingViewingKeySize) {
+        // Bech32 decoding
+        data.reserve((bech.second.size() * 5) / 8);
+        if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, bech.second.begin(), bech.second.end())) {
+            CDataStream ss(data, SER_NETWORK, PROTOCOL_VERSION);
+            libzcash::SaplingIncomingViewingKey ret;
+            ss >> ret;
+            return ret;
+        }
+    }
     return libzcash::InvalidEncoding();
 }
 

src/komodo_bitcoind.h

@@ -207,8 +207,11 @@ try_again:
 
     if ( strncmp(url,"https",5) == 0 )
     {
-        curl_easy_setopt(curl_handle,CURLOPT_SSL_VERIFYPEER,0);
+        
-        curl_easy_setopt(curl_handle,CURLOPT_SSL_VERIFYHOST,0);
+        /* printf("[ Decker ] SSL: %s\n", curl_version()); */
+        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
+        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
+        //curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L); // this is useful for debug, but seems crash on libcurl/7.64.1 OpenSSL/1.1.1b zlib/1.2.8 librtmp/2.3
     }
     if ( userpass != 0 )
         curl_easy_setopt(curl_handle,CURLOPT_USERPWD,	userpass);

src/wallet/gtest/test_wallet.cpp

@@ -526,6 +526,59 @@ TEST(WalletTests, FindMySaplingNotes) {
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
 }
 
+TEST(WalletTests, FindMySaplingNotesWithIvkOnly) {
+    SelectParams(CBaseChainParams::REGTEST);
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
+    auto consensusParams = Params().GetConsensus();
+
+    TestWallet wallet;
+
+    // Generate dummy Sapling address
+    std::vector<unsigned char, secure_allocator<unsigned char>> rawSeed(32);
+    HDSeed seed(rawSeed);
+    auto sk = libzcash::SaplingExtendedSpendingKey::Master(seed);
+    auto addr = sk.DefaultAddress();
+    auto expsk = sk.expsk;
+    auto fvk = expsk.full_viewing_key();
+    auto pk = sk.DefaultAddress();
+    auto ivk = fvk.in_viewing_key();
+
+    // Generate dummy Sapling note
+    libzcash::SaplingNote note(pk, 50000);
+    auto cm = note.cm().get();
+    SaplingMerkleTree tree;
+    tree.append(cm);
+    auto anchor = tree.root();
+    auto witness = tree.witness();
+
+    // Generate transaction
+    auto builder = TransactionBuilder(consensusParams, 1);
+    ASSERT_TRUE(builder.AddSaplingSpend(expsk, note, anchor, witness));
+    builder.AddSaplingOutput(fvk.ovk, pk, 25000, {});
+    auto maybe_tx = builder.Build();
+    ASSERT_EQ(static_cast<bool>(maybe_tx), true);
+    auto tx = maybe_tx.get();
+
+    // No Sapling notes can be found in tx which does not belong to the wallet
+    CWalletTx wtx {&wallet, tx};
+    ASSERT_FALSE(wallet.HaveSaplingSpendingKey(fvk));
+    ASSERT_FALSE(wallet.HaveSaplingIncomingViewingKey(addr));
+    auto noteMap = wallet.FindMySaplingNotes(wtx).first;
+    EXPECT_EQ(0, noteMap.size());
+
+    // Add ivk to wallet, so Sapling notes can be found
+    ASSERT_TRUE(wallet.AddSaplingIncomingViewingKey(ivk, addr));
+    ASSERT_FALSE(wallet.HaveSaplingSpendingKey(fvk));
+    ASSERT_TRUE(wallet.HaveSaplingIncomingViewingKey(addr));
+    noteMap = wallet.FindMySaplingNotes(wtx).first;
+    EXPECT_EQ(2, noteMap.size());
+
+    // Revert to default
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
+    UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
+}
+
 TEST(WalletTests, FindMySproutNotes) {
     CWallet wallet;
 

src/wallet/gtest/test_wallet_zkeys.cpp

@@ -82,6 +82,9 @@ TEST(wallet_zkeys_tests, StoreAndLoadSaplingZkeys) {
     EXPECT_TRUE(wallet.HaveSaplingIncomingViewingKey(sk.DefaultAddress()));
     EXPECT_TRUE(wallet.HaveSaplingIncomingViewingKey(dpa));
 
+    // verify that resets nTimeFirstKey, since there is no birthday info for watch-only keys
+    EXPECT_EQ(wallet.nTimeFirstKey, 1);
+
     // Load a third key into the wallet
     auto sk2 = m.Derive(1);
     ASSERT_TRUE(wallet.LoadSaplingZKey(sk2));

src/wallet/rpcdump.cpp

@@ -694,7 +694,7 @@ UniValue z_importviewingkey(const UniValue& params, bool fHelp)
     if (!EnsureWalletIsAvailable(fHelp))
         return NullUniValue;
 
-    if (fHelp || params.size() < 1 || params.size() > 3)
+    if (fHelp || params.size() < 1 || params.size() > 4)
         throw runtime_error(
             "z_importviewingkey \"vkey\" ( rescan startHeight )\n"
             "\nAdds a viewing key (as returned by z_exportviewingkey) to your wallet.\n"
@@ -702,6 +702,7 @@ UniValue z_importviewingkey(const UniValue& params, bool fHelp)
             "1. \"vkey\"             (string, required) The viewing key (see z_exportviewingkey)\n"
             "2. rescan             (string, optional, default=\"whenkeyisnew\") Rescan the wallet for transactions - can be \"yes\", \"no\" or \"whenkeyisnew\"\n"
             "3. startHeight        (numeric, optional, default=0) Block height to start rescan from\n"
+            "4. zaddr               (string, optional, default=\"\") zaddr in case of importing viewing key for Sapling\n"
             "\nNote: This call can take minutes to complete if rescan is true.\n"
             "\nExamples:\n"
             "\nImport a viewing key\n"
@@ -712,6 +713,8 @@ UniValue z_importviewingkey(const UniValue& params, bool fHelp)
             + HelpExampleCli("z_importviewingkey", "\"vkey\" whenkeyisnew 30000") +
             "\nRe-import the viewing key with longer partial rescan\n"
             + HelpExampleCli("z_importviewingkey", "\"vkey\" yes 20000") +
+            "\nImport the viewing key for Sapling address\n"
+            + HelpExampleCli("z_importviewingkey", "\"vkey\" no 0 \"zaddr\"") +
             "\nAs a JSON-RPC call\n"
             + HelpExampleRpc("z_importviewingkey", "\"vkey\", \"no\"")
         );
@@ -751,14 +754,34 @@ UniValue z_importviewingkey(const UniValue& params, bool fHelp)
     if (!IsValidViewingKey(viewingkey)) {
         throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid viewing key");
     }
-    // TODO: Add Sapling support. For now, return an error to the user.
-    if (boost::get<libzcash::SproutViewingKey>(&viewingkey) == nullptr) {
-        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Currently, only Sprout viewing keys are supported");
-    }
-    auto vkey = boost::get<libzcash::SproutViewingKey>(viewingkey);
-    auto addr = vkey.address();
 
-    {
+    if (boost::get<libzcash::SproutViewingKey>(&viewingkey) == nullptr) {
+        if (params.size() < 4) {
+            throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Missing zaddr for Sapling viewing key.");
+        }
+        string strAddress = params[3].get_str();
+        auto address = DecodePaymentAddress(strAddress);
+        if (!IsValidPaymentAddress(address)) {
+            throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid zaddr");
+        }
+
+        auto addr = boost::get<libzcash::SaplingPaymentAddress>(address);
+        auto ivk = boost::get<libzcash::SaplingIncomingViewingKey>(viewingkey);
+
+        if (pwalletMain->HaveSaplingIncomingViewingKey(addr)) {
+            if (fIgnoreExistingKey) {
+                return NullUniValue;
+            }
+        } else {
+            pwalletMain->MarkDirty();
+
+            if (!pwalletMain->AddSaplingIncomingViewingKey(ivk, addr)) {
+                throw JSONRPCError(RPC_WALLET_ERROR, "Error adding viewing key to wallet");
+            }
+        }
+    } else {
+        auto vkey = boost::get<libzcash::SproutViewingKey>(viewingkey);
+        auto addr = vkey.address();
         if (pwalletMain->HaveSproutSpendingKey(addr)) {
             throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this viewing key");
         }
@@ -775,13 +798,12 @@ UniValue z_importviewingkey(const UniValue& params, bool fHelp)
                 throw JSONRPCError(RPC_WALLET_ERROR, "Error adding viewing key to wallet");
             }
         }
-
-        // We want to scan for transactions and notes
-        if (fRescan) {
-            pwalletMain->ScanForWalletTransactions(chainActive[nRescanHeight], true);
-        }
     }
 
+    // We want to scan for transactions and notes
+    if (fRescan) {
+        pwalletMain->ScanForWalletTransactions(chainActive[nRescanHeight], true);
+    }
     return NullUniValue;
 }
 
@@ -853,12 +875,17 @@ UniValue z_exportviewingkey(const UniValue& params, bool fHelp)
     if (!IsValidPaymentAddress(address)) {
         throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid zaddr");
     }
-    // TODO: Add Sapling support. For now, return an error to the user.
-    if (boost::get<libzcash::SproutPaymentAddress>(&address) == nullptr) {
-        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Currently, only Sprout zaddrs are supported");
-    }
-    auto addr = boost::get<libzcash::SproutPaymentAddress>(address);
 
+    if (boost::get<libzcash::SproutPaymentAddress>(&address) == nullptr) {
+        auto addr = boost::get<libzcash::SaplingPaymentAddress>(address);
+        libzcash::SaplingIncomingViewingKey ivk;
+        if(!pwalletMain->GetSaplingIncomingViewingKey(addr, ivk)) {
+            throw JSONRPCError(RPC_WALLET_ERROR, "Wallet does not hold viewing key for this zaddr");
+        }
+        return EncodeViewingKey(ivk);
+    }
+
+    auto addr = boost::get<libzcash::SproutPaymentAddress>(address);
     libzcash::SproutViewingKey vk;
     if (!pwalletMain->GetSproutViewingKey(addr, vk)) {
         libzcash::SproutSpendingKey k;

src/wallet/rpcwallet.cpp

@@ -3915,7 +3915,8 @@ UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp)
     }
 
     // Visitor to support Sprout and Sapling addrs
-    if (!boost::apply_visitor(PaymentAddressBelongsToWallet(pwalletMain), zaddr)) {
+    if (!boost::apply_visitor(PaymentAddressBelongsToWallet(pwalletMain), zaddr) &&
+        !boost::apply_visitor(IncomingViewingKeyBelongsToWallet(pwalletMain), zaddr)) {
         throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "From address does not belong to this node, zaddr spending key or viewing key not found.");
     }
 

src/wallet/wallet.cpp

@@ -185,6 +185,7 @@ bool CWallet::AddSaplingZKey(
         return false;
     }
 
+    nTimeFirstKey = 1; // No birthday information for viewing keys.
     if (!fFileBacked) {
         return true;
     }
@@ -1541,26 +1542,29 @@ void CWallet::UpdateSaplingNullifierNoteMapWithTx(CWalletTx& wtx) {
         }
         else {
             uint64_t position = nd.witnesses.front().position();
-            SaplingFullViewingKey fvk = mapSaplingFullViewingKeys.at(nd.ivk);
+            // Skip if we only have incoming viewing key
-            OutputDescription output = wtx.vShieldedOutput[op.n];
+            if (mapSaplingFullViewingKeys.count(nd.ivk) != 0) {
-            auto optPlaintext = SaplingNotePlaintext::decrypt(output.encCiphertext, nd.ivk, output.ephemeralKey, output.cm);
+                SaplingFullViewingKey fvk = mapSaplingFullViewingKeys.at(nd.ivk);
-            if (!optPlaintext) {
+                OutputDescription output = wtx.vShieldedOutput[op.n];
-                // An item in mapSaplingNoteData must have already been successfully decrypted,
+                auto optPlaintext = SaplingNotePlaintext::decrypt(output.encCiphertext, nd.ivk, output.ephemeralKey, output.cm);
-                // otherwise the item would not exist in the first place.
+                if (!optPlaintext) {
-                assert(false);
+                    // An item in mapSaplingNoteData must have already been successfully decrypted,
+                    // otherwise the item would not exist in the first place.
+                    assert(false);
+                }
+                auto optNote = optPlaintext.get().note(nd.ivk);
+                if (!optNote) {
+                    assert(false);
+                }
+                auto optNullifier = optNote.get().nullifier(fvk, position);
+                if (!optNullifier) {
+                    // This should not happen.  If it does, maybe the position has been corrupted or miscalculated?
+                    assert(false);
+                }
+                uint256 nullifier = optNullifier.get();
+                mapSaplingNullifiersToNotes[nullifier] = op;
+                item.second.nullifier = nullifier;
             }
-            auto optNote = optPlaintext.get().note(nd.ivk);
-            if (!optNote) {
-                assert(false);
-            }
-            auto optNullifier = optNote.get().nullifier(fvk, position);
-            if (!optNullifier) {
-                // This should not happen.  If it does, maybe the position has been corrupted or miscalculated?
-                assert(false);
-            }
-            uint256 nullifier = optNullifier.get();
-            mapSaplingNullifiersToNotes[nullifier] = op;
-            item.second.nullifier = nullifier;
         }
     }
 }
@@ -1991,23 +1995,40 @@ std::pair<mapSaplingNoteData_t, SaplingIncomingViewingKeyMap> CWallet::FindMySap
     // Protocol Spec: 4.19 Block Chain Scanning (Sapling)
     for (uint32_t i = 0; i < tx.vShieldedOutput.size(); ++i) {
         const OutputDescription output = tx.vShieldedOutput[i];
+        bool found = false;
         for (auto it = mapSaplingFullViewingKeys.begin(); it != mapSaplingFullViewingKeys.end(); ++it) {
             SaplingIncomingViewingKey ivk = it->first;
             auto result = SaplingNotePlaintext::decrypt(output.encCiphertext, ivk, output.ephemeralKey, output.cm);
-            if (!result) {
+            if (result) {
-                continue;
+                auto address = ivk.address(result.get().d);
+                if (address && mapSaplingIncomingViewingKeys.count(address.get()) == 0) {
+                    viewingKeysToAdd[address.get()] = ivk;
+                }
+                // We don't cache the nullifier here as computing it requires knowledge of the note position
+                // in the commitment tree, which can only be determined when the transaction has been mined.
+                SaplingOutPoint op {hash, i};
+                SaplingNoteData nd;
+                nd.ivk = ivk;
+                noteData.insert(std::make_pair(op, nd));
+                found = true;
+                break;
             }
-            auto address = ivk.address(result.get().d);
+        }
-            if (address && mapSaplingIncomingViewingKeys.count(address.get()) == 0) {
+        if (!found) {
-                viewingKeysToAdd[address.get()] = ivk;
+            for (auto it = mapSaplingIncomingViewingKeys.begin(); it != mapSaplingIncomingViewingKeys.end(); ++it) {
+                SaplingIncomingViewingKey ivk = it-> second;
+                auto result = SaplingNotePlaintext::decrypt(output.encCiphertext, ivk, output.ephemeralKey, output.cm);
+                if (!result) {
+                    continue;
+                }
+                // We don't cache the nullifier here as computing it requires knowledge of the note position
+                // in the commitment tree, which can only be determined when the transaction has been mined.
+                SaplingOutPoint op {hash, i};
+                SaplingNoteData nd;
+                nd.ivk = ivk;
+                noteData.insert(std::make_pair(op, nd));
+                break;
             }
-            // We don't cache the nullifier here as computing it requires knowledge of the note position
-            // in the commitment tree, which can only be determined when the transaction has been mined.
-            SaplingOutPoint op {hash, i};
-            SaplingNoteData nd;
-            nd.ivk = ivk;
-            noteData.insert(std::make_pair(op, nd));
-            break;
         }
     }
 
@@ -5071,6 +5092,22 @@ void CWallet::GetFilteredNotes(
 // Shielded key and address generalizations
 //
 
+bool IncomingViewingKeyBelongsToWallet::operator()(const libzcash::SproutPaymentAddress &zaddr) const
+{
+    return m_wallet->HaveSproutViewingKey(zaddr);
+}
+
+bool IncomingViewingKeyBelongsToWallet::operator()(const libzcash::SaplingPaymentAddress &zaddr) const
+{
+    libzcash::SaplingIncomingViewingKey ivk;
+    return m_wallet->GetSaplingIncomingViewingKey(zaddr, ivk);
+}
+
+bool IncomingViewingKeyBelongsToWallet::operator()(const libzcash::InvalidEncoding& no) const
+{
+    return false;
+}
+
 bool PaymentAddressBelongsToWallet::operator()(const libzcash::SproutPaymentAddress &zaddr) const
 {
     return m_wallet->HaveSproutSpendingKey(zaddr) || m_wallet->HaveSproutViewingKey(zaddr);

src/wallet/wallet.h

@@ -1392,6 +1392,19 @@ public:
     bool operator()(const libzcash::InvalidEncoding& no) const;
 };
 
+
+class IncomingViewingKeyBelongsToWallet : public boost::static_visitor<bool>
+{
+private:
+    CWallet *m_wallet;
+public:
+    IncomingViewingKeyBelongsToWallet(CWallet *wallet) : m_wallet(wallet) {}
+
+    bool operator()(const libzcash::SproutPaymentAddress &zaddr) const;
+    bool operator()(const libzcash::SaplingPaymentAddress &zaddr) const;
+    bool operator()(const libzcash::InvalidEncoding& no) const;
+};
+
 class HaveSpendingKeyForPaymentAddress : public boost::static_visitor<bool>
 {
 private:

src/zcash/Address.hpp

@@ -219,7 +219,7 @@ public:
 };
 
 typedef boost::variant<InvalidEncoding, SproutPaymentAddress, SaplingPaymentAddress> PaymentAddress;
-typedef boost::variant<InvalidEncoding, SproutViewingKey> ViewingKey;
+typedef boost::variant<InvalidEncoding, SproutViewingKey, SaplingIncomingViewingKey> ViewingKey;
 
 }