diff --git a/.travis.yml b/.travis.yml index bfc8b8b4a..d7291e03e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,6 @@ matrix: dist: xenial sudo: required env: LINUX_DEPLOY="true" OSX_DEPLOY="false" $TRAVIS_OS_NAME="linux" - - os: osx - osx_image: xcode8 - env: OSX_DEPLOY="true" LINUX_DEPLOY="false" $TRAVIS_OS_NAME="osx" branches: only: - master diff --git a/README.md b/README.md index d7ae47669..71aa7b8a4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/KomodoPlatform/komodo.svg?branch=dev)](https://travis-ci.org/KomodoPlatform/komodo) +[![Build Status](https://travis-ci.org/KomodoPlatform/komodo.svg?branch=master)](https://travis-ci.org/KomodoPlatform/komodo) --- ![Komodo Logo](https://i.imgur.com/E8LtkAa.png "Komodo Logo") diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 6bcf4cd18..8e465287b 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -56,7 +56,7 @@ def sync_blocks(rpc_connections, wait=1): def sync_mempools(rpc_connections, wait=1): """ Wait until everybody has the same transactions in their memory - pools + pools, and has notified all internal listeners of them """ while True: pool = set(rpc_connections[0].getrawmempool()) @@ -68,6 +68,14 @@ def sync_mempools(rpc_connections, wait=1): break time.sleep(wait) + # Now that the mempools are in sync, wait for the internal + # notifications to finish + while True: + notified = [ x.getmempoolinfo()['fullyNotified'] for x in rpc_connections ] + if notified == [ True ] * len(notified): + break + time.sleep(wait) + bitcoind_processes = {} def initialize_datadir(dirname, n): diff --git a/src/init.cpp b/src/init.cpp index 6a3e40450..ae3d9f7d4 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -74,7 +74,9 @@ #include #include #include +#include #include +#include #include @@ -727,6 +729,22 @@ void ThreadImport(std::vector vImportFiles) } } +void ThreadNotifyRecentlyAdded() +{ + while (true) { + // Run the notifier on an integer second in the steady clock. + auto now = std::chrono::steady_clock::now().time_since_epoch(); + auto nextFire = std::chrono::duration_cast( + now + std::chrono::seconds(1)); + std::this_thread::sleep_until( + std::chrono::time_point(nextFire)); + + boost::this_thread::interruption_point(); + + mempool.NotifyRecentlyAdded(); + } +} + /** Sanity checks * Ensure that Bitcoin is running in a usable environment with all * necessary library support. @@ -1964,6 +1982,10 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif + // Start the thread that notifies listeners of transactions that have been + // recently added to the mempool. + threadGroup.create_thread(boost::bind(&TraceThread, "txnotify", &ThreadNotifyRecentlyAdded)); + if (GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) StartTorControl(threadGroup, scheduler); diff --git a/src/main.cpp b/src/main.cpp index da8b706ea..a57686a7e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2080,8 +2080,6 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } } - SyncWithWallets(tx, NULL); - return true; } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 881dc1b66..59691b51d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1906,6 +1906,10 @@ UniValue mempoolInfoToJSON() ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); + if (Params().NetworkIDString() == "regtest") { + ret.push_back(Pair("fullyNotified", mempool.IsFullyNotified())); + } + return ret; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 5f0fb2a47..84d416f96 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -29,6 +29,7 @@ #include "timedata.h" #include "util.h" #include "utilmoneystr.h" +#include "validationinterface.h" #include "version.h" #define _COINBASE_MATURITY 100 @@ -119,6 +120,8 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, LOCK(cs); mapTx.insert(entry); const CTransaction& tx = mapTx.find(hash)->GetTx(); + mapRecentlyAddedTx[tx.GetHash()] = &tx; + nRecentlyAddedSequence += 1; if (!tx.IsCoinImport()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { @@ -365,6 +368,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem txToRemove.push_back(it->second.ptx->GetHash()); } } + mapRecentlyAddedTx.erase(hash); BOOST_FOREACH(const CTxIn& txin, tx.vin) mapNextTx.erase(txin.prevout); BOOST_FOREACH(const JSDescription& joinsplit, tx.vjoinsplit) { @@ -838,6 +842,49 @@ bool CTxMemPool::nullifierExists(const uint256& nullifier, ShieldedType type) co } } +void CTxMemPool::NotifyRecentlyAdded() +{ + uint64_t recentlyAddedSequence; + std::vector txs; + { + LOCK(cs); + recentlyAddedSequence = nRecentlyAddedSequence; + for (const auto& kv : mapRecentlyAddedTx) { + txs.push_back(*(kv.second)); + } + mapRecentlyAddedTx.clear(); + } + + // A race condition can occur here between these SyncWithWallets calls, and + // the ones triggered by block logic (in ConnectTip and DisconnectTip). It + // is harmless because calling SyncWithWallets(_, NULL) does not alter the + // wallet transaction's block information. + for (auto tx : txs) { + try { + SyncWithWallets(tx, NULL); + } catch (const boost::thread_interrupted&) { + throw; + } catch (const std::exception& e) { + PrintExceptionContinue(&e, "CTxMemPool::NotifyRecentlyAdded()"); + } catch (...) { + PrintExceptionContinue(NULL, "CTxMemPool::NotifyRecentlyAdded()"); + } + } + + // Update the notified sequence number. We only need this in regtest mode, + // and should not lock on cs after calling SyncWithWallets otherwise. + if (Params().NetworkIDString() == "regtest") { + LOCK(cs); + nNotifiedSequence = recentlyAddedSequence; + } +} + +bool CTxMemPool::IsFullyNotified() { + assert(Params().NetworkIDString() == "regtest"); + LOCK(cs); + return nRecentlyAddedSequence == nNotifiedSequence; +} + CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView *baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } bool CCoinsViewMemPool::GetNullifier(const uint256 &nf, ShieldedType type) const diff --git a/src/txmempool.h b/src/txmempool.h index d3e7f7b57..b73ff4b39 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -148,6 +148,10 @@ private: uint64_t totalTxSize = 0; //! sum of all mempool tx' byte sizes uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves) + std::map mapRecentlyAddedTx; + uint64_t nRecentlyAddedSequence = 0; + uint64_t nNotifiedSequence = 0; + std::map mapSproutNullifiers; std::map mapSaplingNullifiers; @@ -234,6 +238,9 @@ public: bool nullifierExists(const uint256& nullifier, ShieldedType type) const; + void NotifyRecentlyAdded(); + bool IsFullyNotified(); + unsigned long size() { LOCK(cs); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 4d834ccee..77ea2e59d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1841,7 +1841,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl void CWallet::SyncTransaction(const CTransaction& tx, const CBlock* pblock) { - LOCK2(cs_main, cs_wallet); + LOCK(cs_wallet); if (!AddToWalletIfInvolvingMe(tx, pblock, true)) return; // Not one of ours @@ -4830,9 +4830,8 @@ CWalletKey::CWalletKey(int64_t nExpires) nTimeExpires = nExpires; } -int CMerkleTx::SetMerkleBranch(const CBlock& block) +void CMerkleTx::SetMerkleBranch(const CBlock& block) { - AssertLockHeld(cs_main); CBlock blockTmp; // Update the tx's hashBlock @@ -4847,21 +4846,10 @@ int CMerkleTx::SetMerkleBranch(const CBlock& block) vMerkleBranch.clear(); nIndex = -1; LogPrintf("ERROR: SetMerkleBranch(): couldn't find tx in block\n"); - return 0; } // Fill in merkle branch vMerkleBranch = block.GetMerkleBranch(nIndex); - - // Is the tx in a block that's in the main chain - BlockMap::iterator mi = mapBlockIndex.find(hashBlock); - if (mi == mapBlockIndex.end()) - return 0; - const CBlockIndex* pindex = (*mi).second; - if (!pindex || !chainActive.Contains(pindex)) - return 0; - - return chainActive.Height() - pindex->GetHeight() + 1; } int CMerkleTx::GetDepthInMainChainINTERNAL(const CBlockIndex* &pindexRet) const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 296e2fa57..60b5fb3bf 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -386,7 +386,7 @@ public: READWRITE(nIndex); } - int SetMerkleBranch(const CBlock& block); + void SetMerkleBranch(const CBlock& block); /** diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index ee8f7b641..23210c784 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -173,15 +173,21 @@ boost::optional SaplingOutgoingPlaintext::decrypt( } // Deserialize from the plaintext - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); + try { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); - SaplingOutgoingPlaintext ret; - ss >> ret; + SaplingOutgoingPlaintext ret; + ss >> ret; - assert(ss.size() == 0); + assert(ss.size() == 0); - return ret; + return ret; + } catch (const boost::thread_interrupted&) { + throw; + } catch (...) { + return boost::none; + } } boost::optional SaplingNotePlaintext::decrypt( @@ -197,13 +203,17 @@ boost::optional SaplingNotePlaintext::decrypt( } // Deserialize from the plaintext - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - SaplingNotePlaintext ret; - ss >> ret; - - assert(ss.size() == 0); + try { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); + ss >> ret; + assert(ss.size() == 0); + } catch (const boost::thread_interrupted&) { + throw; + } catch (...) { + return boost::none; + } uint256 pk_d; if (!librustzcash_ivk_to_pkd(ivk.begin(), ret.d.data(), pk_d.begin())) { @@ -243,11 +253,17 @@ boost::optional SaplingNotePlaintext::decrypt( } // Deserialize from the plaintext - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - SaplingNotePlaintext ret; - ss >> ret; + try { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); + ss >> ret; + assert(ss.size() == 0); + } catch (const boost::thread_interrupted&) { + throw; + } catch (...) { + return boost::none; + } uint256 cmu_expected; if (!librustzcash_sapling_compute_cm( @@ -265,8 +281,6 @@ boost::optional SaplingNotePlaintext::decrypt( return boost::none; } - assert(ss.size() == 0); - return ret; } diff --git a/zcutil/fetch-params.bat b/zcutil/fetch-params.bat index 8b14b1845..f9e70d481 100644 --- a/zcutil/fetch-params.bat +++ b/zcutil/fetch-params.bat @@ -1,28 +1,28 @@ @echo off call :GET_CURRENT_DIR cd %THIS_DIR% -IF NOT EXIST %APPDATA%\ZcashParams ( -MKDIR %APPDATA%\ZcashParams +IF NOT EXIST "%APPDATA%"\ZcashParams ( +MKDIR "%APPDATA%"\ZcashParams ) -IF NOT EXIST %APPDATA%\ZcashParams\sprout-proving.key ( +IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-proving.key ( ECHO Downloading Zcash trusted setup sprout-proving.key, this may take a while ... - .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-proving.key -O %APPDATA%\ZcashParams\sprout-proving.key + .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-proving.key -O "%APPDATA%"\ZcashParams\sprout-proving.key ) -IF NOT EXIST %APPDATA%\ZcashParams\sprout-verifying.key ( +IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-verifying.key ( ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ... - .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-verifying.key -O %APPDATA%\ZcashParams\sprout-verifying.key + .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-verifying.key -O "%APPDATA%"\ZcashParams\sprout-verifying.key ) -IF NOT EXIST %APPDATA%\ZcashParams\sapling-spend.params ( +IF NOT EXIST "%APPDATA%"\ZcashParams\sapling-spend.params ( ECHO Downloading Zcash trusted setup sprout-proving.key, this may take a while ... - .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-spend.params -O %APPDATA%\ZcashParams\sapling-spend.params + .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-spend.params -O "%APPDATA%"\ZcashParams\sapling-spend.params ) -IF NOT EXIST %APPDATA%\ZcashParams\sapling-output.params ( +IF NOT EXIST "%APPDATA%"\ZcashParams\sapling-output.params ( ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ... - .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-output.params -O %APPDATA%\ZcashParams\sapling-output.params + .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sapling-output.params -O "%APPDATA%"\ZcashParams\sapling-output.params ) -IF NOT EXIST %APPDATA%\ZcashParams\sprout-groth16.params ( +IF NOT EXIST "%APPDATA%"\ZcashParams\sprout-groth16.params ( ECHO Downloading Zcash trusted setup sprout-verifying.key, this may take a while ... - .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-groth16.params -O %APPDATA%\ZcashParams\sprout-groth16.params + .\wget64.exe --progress=dot:giga --continue --retry-connrefused --waitretry=3 --timeout=30 https://z.cash/downloads/sprout-groth16.params -O "%APPDATA%"\ZcashParams\sprout-groth16.params ) goto :EOF :GET_CURRENT_DIR