Merge branch 'dev' into FSM

.travis.yml

@@ -6,9 +6,6 @@ matrix:
       dist: xenial
       sudo: required
       env: LINUX_DEPLOY="true" OSX_DEPLOY="false" $TRAVIS_OS_NAME="linux"
-    - os: osx
-      osx_image: xcode8
-      env: OSX_DEPLOY="true" LINUX_DEPLOY="false" $TRAVIS_OS_NAME="osx"
 branches:
   only:
     - master

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/KomodoPlatform/komodo.svg?branch=dev)](https://travis-ci.org/KomodoPlatform/komodo)
+[![Build Status](https://travis-ci.org/KomodoPlatform/komodo.svg?branch=master)](https://travis-ci.org/KomodoPlatform/komodo)
 ---
 ![Komodo Logo](https://i.imgur.com/E8LtkAa.png "Komodo Logo")
 

qa/rpc-tests/test_framework/util.py

@@ -56,7 +56,7 @@ def sync_blocks(rpc_connections, wait=1):
 def sync_mempools(rpc_connections, wait=1):
     """
     Wait until everybody has the same transactions in their memory
-    pools
+    pools, and has notified all internal listeners of them
     """
     while True:
         pool = set(rpc_connections[0].getrawmempool())
@@ -68,6 +68,14 @@ def sync_mempools(rpc_connections, wait=1):
             break
         time.sleep(wait)
 
+    # Now that the mempools are in sync, wait for the internal
+    # notifications to finish
+    while True:
+        notified = [ x.getmempoolinfo()['fullyNotified'] for x in rpc_connections ]
+        if notified == [ True ] * len(notified):
+            break
+        time.sleep(wait)
+
 bitcoind_processes = {}
 
 def initialize_datadir(dirname, n):

src/init.cpp

@@ -74,7 +74,9 @@
 #include <boost/function.hpp>
 #include <boost/interprocess/sync/file_lock.hpp>
 #include <boost/thread.hpp>
+#include <chrono>
 #include <openssl/crypto.h>
+#include <thread>
 
 #include <libsnark/common/profiling.hpp>
 
@@ -727,6 +729,22 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
     }
 }
 
+void ThreadNotifyRecentlyAdded()
+{
+    while (true) {
+        // Run the notifier on an integer second in the steady clock.
+        auto now = std::chrono::steady_clock::now().time_since_epoch();
+        auto nextFire = std::chrono::duration_cast<std::chrono::seconds>(
+            now + std::chrono::seconds(1));
+        std::this_thread::sleep_until(
+            std::chrono::time_point<std::chrono::steady_clock>(nextFire));
+
+        boost::this_thread::interruption_point();
+
+        mempool.NotifyRecentlyAdded();
+    }
+}
+
 /** Sanity checks
  *  Ensure that Bitcoin is running in a usable environment with all
  *  necessary library support.
@@ -1964,6 +1982,10 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
     LogPrintf("mapAddressBook.size() = %u\n",  pwalletMain ? pwalletMain->mapAddressBook.size() : 0);
 #endif
 
+    // Start the thread that notifies listeners of transactions that have been
+    // recently added to the mempool.
+    threadGroup.create_thread(boost::bind(&TraceThread<void (*)()>, "txnotify", &ThreadNotifyRecentlyAdded));
+
     if (GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION))
         StartTorControl(threadGroup, scheduler);
 

src/main.cpp

@@ -2080,8 +2080,6 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
         }
     }
 
-    SyncWithWallets(tx, NULL);
-
     return true;
 }
 

src/rpc/blockchain.cpp

@@ -1906,6 +1906,10 @@ UniValue mempoolInfoToJSON()
     ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize()));
     ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage()));
 
+    if (Params().NetworkIDString() == "regtest") {
+        ret.push_back(Pair("fullyNotified", mempool.IsFullyNotified()));
+    }
+    
     return ret;
 }
 

src/txmempool.cpp

@@ -29,6 +29,7 @@
 #include "timedata.h"
 #include "util.h"
 #include "utilmoneystr.h"
+#include "validationinterface.h"
 #include "version.h"
 #define _COINBASE_MATURITY 100
 
@@ -119,6 +120,8 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry,
     LOCK(cs);
     mapTx.insert(entry);
     const CTransaction& tx = mapTx.find(hash)->GetTx();
+    mapRecentlyAddedTx[tx.GetHash()] = &tx;
+    nRecentlyAddedSequence += 1;
     if (!tx.IsCoinImport()) {
         for (unsigned int i = 0; i < tx.vin.size(); i++)
         {
@@ -365,6 +368,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem
                     txToRemove.push_back(it->second.ptx->GetHash());
                 }
             }
+            mapRecentlyAddedTx.erase(hash);
             BOOST_FOREACH(const CTxIn& txin, tx.vin)
                 mapNextTx.erase(txin.prevout);
             BOOST_FOREACH(const JSDescription& joinsplit, tx.vjoinsplit) {
@@ -838,6 +842,49 @@ bool CTxMemPool::nullifierExists(const uint256& nullifier, ShieldedType type) co
     }
 }
 
+void CTxMemPool::NotifyRecentlyAdded()
+{
+    uint64_t recentlyAddedSequence;
+    std::vector<CTransaction> txs;
+    {
+        LOCK(cs);
+        recentlyAddedSequence = nRecentlyAddedSequence;
+        for (const auto& kv : mapRecentlyAddedTx) {
+            txs.push_back(*(kv.second));
+        }
+        mapRecentlyAddedTx.clear();
+    }
+
+    // A race condition can occur here between these SyncWithWallets calls, and
+    // the ones triggered by block logic (in ConnectTip and DisconnectTip). It
+    // is harmless because calling SyncWithWallets(_, NULL) does not alter the
+    // wallet transaction's block information.
+    for (auto tx : txs) {
+        try {
+            SyncWithWallets(tx, NULL);
+        } catch (const boost::thread_interrupted&) {
+            throw;
+        } catch (const std::exception& e) {
+            PrintExceptionContinue(&e, "CTxMemPool::NotifyRecentlyAdded()");
+        } catch (...) {
+            PrintExceptionContinue(NULL, "CTxMemPool::NotifyRecentlyAdded()");
+        }
+    }
+
+    // Update the notified sequence number. We only need this in regtest mode,
+    // and should not lock on cs after calling SyncWithWallets otherwise.
+    if (Params().NetworkIDString() == "regtest") {
+        LOCK(cs);
+        nNotifiedSequence = recentlyAddedSequence;
+    }
+}
+
+bool CTxMemPool::IsFullyNotified() {
+    assert(Params().NetworkIDString() == "regtest");
+    LOCK(cs);
+    return nRecentlyAddedSequence == nNotifiedSequence;
+}
+
 CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { }
 
 bool CCoinsViewMemPool::GetNullifier(const uint256 &nf, ShieldedType type) const

src/txmempool.h

@@ -148,6 +148,10 @@ private:
     uint64_t totalTxSize = 0; //! sum of all mempool tx' byte sizes
     uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves)
 
+    std::map<uint256, const CTransaction*> mapRecentlyAddedTx;
+    uint64_t nRecentlyAddedSequence = 0;
+    uint64_t nNotifiedSequence = 0;
+
     std::map<uint256, const CTransaction*> mapSproutNullifiers;
     std::map<uint256, const CTransaction*> mapSaplingNullifiers;
 
@@ -234,6 +238,9 @@ public:
 
     bool nullifierExists(const uint256& nullifier, ShieldedType type) const;
 
+    void NotifyRecentlyAdded();
+    bool IsFullyNotified();
+    
     unsigned long size()
     {
         LOCK(cs);

src/wallet/wallet.cpp

@@ -1841,7 +1841,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl
 
 void CWallet::SyncTransaction(const CTransaction& tx, const CBlock* pblock)
 {
-    LOCK2(cs_main, cs_wallet);
+    LOCK(cs_wallet);
     if (!AddToWalletIfInvolvingMe(tx, pblock, true))
         return; // Not one of ours
 
@@ -4830,9 +4830,8 @@ CWalletKey::CWalletKey(int64_t nExpires)
     nTimeExpires = nExpires;
 }
 
-int CMerkleTx::SetMerkleBranch(const CBlock& block)
+void CMerkleTx::SetMerkleBranch(const CBlock& block)
 {
-    AssertLockHeld(cs_main);
     CBlock blockTmp;
 
     // Update the tx's hashBlock
@@ -4847,21 +4846,10 @@ int CMerkleTx::SetMerkleBranch(const CBlock& block)
         vMerkleBranch.clear();
         nIndex = -1;
         LogPrintf("ERROR: SetMerkleBranch(): couldn't find tx in block\n");
-        return 0;
     }
 
     // Fill in merkle branch
     vMerkleBranch = block.GetMerkleBranch(nIndex);
-
-    // Is the tx in a block that's in the main chain
-    BlockMap::iterator mi = mapBlockIndex.find(hashBlock);
-    if (mi == mapBlockIndex.end())
-        return 0;
-    const CBlockIndex* pindex = (*mi).second;
-    if (!pindex || !chainActive.Contains(pindex))
-        return 0;
-
-    return chainActive.Height() - pindex->GetHeight() + 1;
 }
 
 int CMerkleTx::GetDepthInMainChainINTERNAL(const CBlockIndex* &pindexRet) const

src/wallet/wallet.h

@@ -386,7 +386,7 @@ public:
         READWRITE(nIndex);
     }
 
-    int SetMerkleBranch(const CBlock& block);
+    void SetMerkleBranch(const CBlock& block);
 
 
     /**

src/zcash/Note.cpp

@@ -173,15 +173,21 @@ boost::optional<SaplingOutgoingPlaintext> SaplingOutgoingPlaintext::decrypt(
     }
 
     // Deserialize from the plaintext
-    CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
+    try {
-    ss << pt.get();
+        CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
+        ss << pt.get();
 
-    SaplingOutgoingPlaintext ret;
+        SaplingOutgoingPlaintext ret;
-    ss >> ret;
+        ss >> ret;
 
-    assert(ss.size() == 0);
+        assert(ss.size() == 0);
 
-    return ret;
+        return ret;
+    } catch (const boost::thread_interrupted&) {
+        throw;
+    } catch (...) {
+        return boost::none;
+    }
 }
 
 boost::optional<SaplingNotePlaintext> SaplingNotePlaintext::decrypt(
@@ -197,13 +203,17 @@ boost::optional<SaplingNotePlaintext> SaplingNotePlaintext::decrypt(
     }
 
     // Deserialize from the plaintext
-    CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
-    ss << pt.get();
-
     SaplingNotePlaintext ret;
-    ss >> ret;
+    try {
-
+        CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
-    assert(ss.size() == 0);
+        ss << pt.get();
+        ss >> ret;
+        assert(ss.size() == 0);
+    } catch (const boost::thread_interrupted&) {
+        throw;
+    } catch (...) {
+        return boost::none;
+    }
 
     uint256 pk_d;
     if (!librustzcash_ivk_to_pkd(ivk.begin(), ret.d.data(), pk_d.begin())) {
@@ -243,11 +253,17 @@ boost::optional<SaplingNotePlaintext> SaplingNotePlaintext::decrypt(
     }
 
     // Deserialize from the plaintext
-    CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
-    ss << pt.get();
-
     SaplingNotePlaintext ret;
-    ss >> ret;
+    try {
+        CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
+        ss << pt.get();
+        ss >> ret;
+        assert(ss.size() == 0);
+    } catch (const boost::thread_interrupted&) {
+        throw;
+    } catch (...) {
+        return boost::none;
+    }
 
     uint256 cmu_expected;
     if (!librustzcash_sapling_compute_cm(
@@ -265,8 +281,6 @@ boost::optional<SaplingNotePlaintext> SaplingNotePlaintext::decrypt(
         return boost::none;
     }
 
-    assert(ss.size() == 0);
-
     return ret;
 }
 

zcutil/fetch-params.bat

@@ -1,28 +1,28 @@
 @echo off
 call :GET_CURRENT_DIR
 cd %THIS_DIR%
-IF NOT EXIST %APPDATA%\ZcashParams (
+IF NOT EXIST "%APPDATA%"\ZcashParams (
-MKDIR %APPDATA%\ZcashParams
+MKDIR "%APPDATA%"\ZcashParams
 )
-IF NOT EXIST %APPDATA%\ZcashParams\sprout-proving.key (
+IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-proving.key (
     ECHO Downloading Zcash trusted setup sprout-proving.key, this may take a while ...
-    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-proving.key -O %APPDATA%\ZcashParams\sprout-proving.key
+    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-proving.key -O "%APPDATA%"\ZcashParams\sprout-proving.key
 )
-IF NOT EXIST %APPDATA%\ZcashParams\sprout-verifying.key (
+IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-verifying.key (
     ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ...
-    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-verifying.key -O %APPDATA%\ZcashParams\sprout-verifying.key
+    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-verifying.key -O "%APPDATA%"\ZcashParams\sprout-verifying.key
 )
-IF NOT EXIST %APPDATA%\ZcashParams\sapling-spend.params (
+IF NOT EXIST "%APPDATA%"\ZcashParams\sapling-spend.params (
     ECHO Downloading Zcash trusted setup sprout-proving.key, this may take a while ...
-    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-spend.params -O %APPDATA%\ZcashParams\sapling-spend.params
+    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-spend.params -O "%APPDATA%"\ZcashParams\sapling-spend.params
 )
-IF NOT EXIST %APPDATA%\ZcashParams\sapling-output.params (
+IF NOT EXIST "%APPDATA%"\ZcashParams\sapling-output.params (
     ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ...
-    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-output.params -O %APPDATA%\ZcashParams\sapling-output.params
+    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-output.params -O "%APPDATA%"\ZcashParams\sapling-output.params
 )
-IF NOT EXIST %APPDATA%\ZcashParams\sprout-groth16.params (
+IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-groth16.params (
     ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ...
-    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-groth16.params -O %APPDATA%\ZcashParams\sprout-groth16.params
+    .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-groth16.params -O "%APPDATA%"\ZcashParams\sprout-groth16.params
 )
 goto :EOF
 :GET_CURRENT_DIR