From 6fb8d0c2d64583a9a5b852bd014f63a11ea89eff Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 23 Nov 2016 17:04:20 +1300 Subject: [PATCH 01/56] Skip JoinSplit verification before the last checkpoint Part of #1749 --- src/gtest/test_checkblock.cpp | 5 +++- src/main.cpp | 43 ++++++++++++++++++++++------------ src/main.h | 14 ++++++++--- src/test/checkblock_tests.cpp | 4 +++- src/test/transaction_tests.cpp | 29 +++++++++++++---------- src/wallet/walletdb.cpp | 4 +++- 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/gtest/test_checkblock.cpp b/src/gtest/test_checkblock.cpp index eab73afad..edd8a617d 100644 --- a/src/gtest/test_checkblock.cpp +++ b/src/gtest/test_checkblock.cpp @@ -3,6 +3,7 @@ #include "consensus/validation.h" #include "main.h" +#include "zcash/Proof.hpp" class MockCValidationState : public CValidationState { public: @@ -22,12 +23,14 @@ public: }; TEST(CheckBlock, VersionTooLow) { + auto verifier = libzcash::ProofVerifier::Strict(); + CBlock block; block.nVersion = 1; MockCValidationState state; EXPECT_CALL(state, DoS(100, false, REJECT_INVALID, "version-too-low", false)).Times(1); - EXPECT_FALSE(CheckBlock(block, state, false, false)); + EXPECT_FALSE(CheckBlock(block, state, verifier, false, false)); } TEST(ContextualCheckBlock, BadCoinbaseHeight) { diff --git a/src/main.cpp b/src/main.cpp index 491714f95..a843724ac 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -833,7 +833,8 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in return nSigOps; } -bool CheckTransaction(const CTransaction& tx, CValidationState &state) +bool CheckTransaction(const CTransaction& tx, CValidationState &state, + libzcash::ProofVerifier& verifier) { // Don't count coinbase transactions because mining skews the count if (!tx.IsCoinBase()) { @@ -844,7 +845,6 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) return false; } else { // Ensure that zk-SNARKs verify - auto verifier = libzcash::ProofVerifier::Strict(); BOOST_FOREACH(const JSDescription &joinsplit, tx.vjoinsplit) { if (!joinsplit.Verify(*pzcashParams, verifier, tx.joinSplitPubKey)) { return state.DoS(100, error("CheckTransaction(): joinsplit does not verify"), @@ -1056,7 +1056,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa if (pfMissingInputs) *pfMissingInputs = false; - if (!CheckTransaction(tx, state)) + auto verifier = libzcash::ProofVerifier::Strict(); + if (!CheckTransaction(tx, state, verifier)) return error("AcceptToMemoryPool: CheckTransaction failed"); // Coinbase is only valid in a block, not as a loose transaction @@ -2041,8 +2042,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin { const CChainParams& chainparams = Params(); AssertLockHeld(cs_main); - // Check it again in case a previous version let a bad block in - if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) + + bool fExpensiveChecks = (!fCheckpointsEnabled || pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(chainparams.Checkpoints())); + auto verifier = libzcash::ProofVerifier::Strict(); + auto disabledVerifier = libzcash::ProofVerifier::Disabled(); + + // Check it again to verify JoinSplit proofs, and in case a previous version let a bad block in + if (!CheckBlock(block, state, fExpensiveChecks ? verifier : disabledVerifier, !fJustCheck, !fJustCheck)) return false; // verify that the view's current state corresponds to the previous block @@ -2061,8 +2067,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return true; } - bool fScriptChecks = (!fCheckpointsEnabled || pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(chainparams.Checkpoints())); - // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. BOOST_FOREACH(const CTransaction& tx, block.vtx) { @@ -2088,7 +2092,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CBlockUndo blockundo; - CCheckQueueControl control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); + CCheckQueueControl control(fExpensiveChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); int64_t nTimeStart = GetTimeMicros(); CAmount nFees = 0; @@ -2149,7 +2153,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin nFees += view.GetValueIn(tx)-tx.GetValueOut(); std::vector vChecks; - if (!ContextualCheckInputs(tx, state, view, fScriptChecks, flags, false, chainparams.GetConsensus(), nScriptCheckThreads ? &vChecks : NULL)) + if (!ContextualCheckInputs(tx, state, view, fExpensiveChecks, flags, false, chainparams.GetConsensus(), nScriptCheckThreads ? &vChecks : NULL)) return false; control.Add(vChecks); } @@ -2976,7 +2980,9 @@ bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool f return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, CValidationState& state, + libzcash::ProofVerifier& verifier, + bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. @@ -3021,7 +3027,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Check transactions BOOST_FOREACH(const CTransaction& tx, block.vtx) - if (!CheckTransaction(tx, state)) + if (!CheckTransaction(tx, state, verifier)) return error("CheckBlock(): CheckTransaction failed"); unsigned int nSigOps = 0; @@ -3209,7 +3215,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, if (fTooFarAhead) return true; // Block height is too high } - if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { + // See method docstring for why this is always disabled + auto verifier = libzcash::ProofVerifier::Disabled(); + if ((!CheckBlock(block, state, verifier)) || !ContextualCheckBlock(block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); @@ -3258,7 +3266,8 @@ static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned bool ProcessNewBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, bool fForceProcessing, CDiskBlockPos *dbp) { // Preliminary checks - bool checked = CheckBlock(*pblock, state); + auto verifier = libzcash::ProofVerifier::Disabled(); + bool checked = CheckBlock(*pblock, state, verifier); { LOCK(cs_main); @@ -3294,11 +3303,13 @@ bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; + // JoinSplit proofs are verified in ConnectBlock + auto verifier = libzcash::ProofVerifier::Disabled(); // NOTE: CheckBlockHeader is called by CheckBlock if (!ContextualCheckBlockHeader(block, state, pindexPrev)) return false; - if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) + if (!CheckBlock(block, state, verifier, fCheckPOW, fCheckMerkleRoot)) return false; if (!ContextualCheckBlock(block, state, pindexPrev)) return false; @@ -3619,6 +3630,8 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth CBlockIndex* pindexFailure = NULL; int nGoodTransactions = 0; CValidationState state; + // No need to verify JoinSplits twice + auto verifier = libzcash::ProofVerifier::Disabled(); for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); @@ -3630,7 +3643,7 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth if (!ReadBlockFromDisk(block, pindex)) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 1: verify block validity - if (nCheckLevel >= 1 && !CheckBlock(block, state)) + if (nCheckLevel >= 1 && !CheckBlock(block, state, verifier)) return error("VerifyDB(): *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { diff --git a/src/main.h b/src/main.h index 66808a57a..fd99d28e5 100644 --- a/src/main.h +++ b/src/main.h @@ -339,7 +339,7 @@ bool NonContextualCheckInputs(const CTransaction& tx, CValidationState &state, c void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, int nHeight); /** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state); +bool CheckTransaction(const CTransaction& tx, CValidationState& state, libzcash::ProofVerifier& verifier); bool CheckTransactionWithoutProofVerification(const CTransaction& tx, CValidationState &state); /** Check for standard transaction types @@ -416,7 +416,9 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin /** Context-independent validity checks */ bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); +bool CheckBlock(const CBlock& block, CValidationState& state, + libzcash::ProofVerifier& verifier, + bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Context-dependent validity checks */ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex *pindexPrev); @@ -425,7 +427,13 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex *pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); -/** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ +/** + * Store block on disk. + * JoinSplit proofs are never verified, because: + * - AcceptBlock doesn't perform script checks either. + * - The only caller of AcceptBlock verifies JoinSplit proofs elsewhere. + * If dbp is non-NULL, the file is known to already reside on disk + */ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex **pindex, bool fRequested, CDiskBlockPos* dbp); bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex **ppindex= NULL); diff --git a/src/test/checkblock_tests.cpp b/src/test/checkblock_tests.cpp index 51530c4de..c813c9af9 100644 --- a/src/test/checkblock_tests.cpp +++ b/src/test/checkblock_tests.cpp @@ -7,6 +7,7 @@ #include "main.h" #include "test/test_bitcoin.h" #include "utiltime.h" +#include "zcash/Proof.hpp" #include @@ -56,7 +57,8 @@ BOOST_AUTO_TEST_CASE(May15) // After May 15'th, big blocks are OK: forkingBlock.nTime = tMay15; // Invalidates PoW - BOOST_CHECK(CheckBlock(forkingBlock, state, false, false)); + auto verifier = libzcash::ProofVerifier::Strict(); + BOOST_CHECK(CheckBlock(forkingBlock, state, verifier, false, false)); } SetMockTime(0); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 86d59e809..64fa4b298 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -29,6 +29,7 @@ #include "zcash/Note.hpp" #include "zcash/Address.hpp" +#include "zcash/Proof.hpp" using namespace std; using namespace json_spirit; @@ -97,6 +98,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" Array tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid))); + auto verifier = libzcash::ProofVerifier::Strict(); ScriptError err; BOOST_FOREACH(Value& tv, tests) { @@ -141,7 +143,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) stream >> tx; CValidationState state; - BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); + BOOST_CHECK_MESSAGE(CheckTransaction(tx, state, verifier), strTest); BOOST_CHECK(state.IsValid()); for (unsigned int i = 0; i < tx.vin.size(); i++) @@ -173,6 +175,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" Array tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid))); + auto verifier = libzcash::ProofVerifier::Strict(); ScriptError err; BOOST_FOREACH(Value& tv, tests) { @@ -217,7 +220,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) stream >> tx; CValidationState state; - fValid = CheckTransaction(tx, state) && state.IsValid(); + fValid = CheckTransaction(tx, state, verifier) && state.IsValid(); for (unsigned int i = 0; i < tx.vin.size() && fValid; i++) { @@ -246,11 +249,12 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) CMutableTransaction tx; stream >> tx; CValidationState state; - BOOST_CHECK_MESSAGE(CheckTransaction(tx, state) && state.IsValid(), "Simple deserialized transaction should be valid."); + auto verifier = libzcash::ProofVerifier::Strict(); + BOOST_CHECK_MESSAGE(CheckTransaction(tx, state, verifier) && state.IsValid(), "Simple deserialized transaction should be valid."); // Check that duplicate txins fail tx.vin.push_back(tx.vin[0]); - BOOST_CHECK_MESSAGE(!CheckTransaction(tx, state) || !state.IsValid(), "Transaction with duplicate txins should be invalid."); + BOOST_CHECK_MESSAGE(!CheckTransaction(tx, state, verifier) || !state.IsValid(), "Transaction with duplicate txins should be invalid."); } // @@ -373,6 +377,7 @@ BOOST_AUTO_TEST_CASE(test_basic_joinsplit_verification) BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) { + auto verifier = libzcash::ProofVerifier::Strict(); CMutableTransaction tx; tx.nVersion = 2; { @@ -424,23 +429,23 @@ BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) JSDescription *jsdesc = &newTx.vjoinsplit[0]; jsdesc->vpub_old = -1; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-txns-vpub_old-negative"); jsdesc->vpub_old = MAX_MONEY + 1; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-txns-vpub_old-toolarge"); jsdesc->vpub_old = 0; jsdesc->vpub_new = -1; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-txns-vpub_new-negative"); jsdesc->vpub_new = MAX_MONEY + 1; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-txns-vpub_new-toolarge"); jsdesc->vpub_new = (MAX_MONEY / 2) + 10; @@ -450,7 +455,7 @@ BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) JSDescription *jsdesc2 = &newTx.vjoinsplit[1]; jsdesc2->vpub_new = (MAX_MONEY / 2) + 10; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-txns-txintotal-toolarge"); } { @@ -464,7 +469,7 @@ BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) jsdesc->nullifiers[0] = GetRandHash(); jsdesc->nullifiers[1] = jsdesc->nullifiers[0]; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-joinsplits-nullifiers-duplicate"); jsdesc->nullifiers[1] = GetRandHash(); @@ -475,7 +480,7 @@ BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) jsdesc2->nullifiers[0] = GetRandHash(); jsdesc2->nullifiers[1] = jsdesc->nullifiers[0]; - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-joinsplits-nullifiers-duplicate"); } { @@ -494,7 +499,7 @@ BOOST_AUTO_TEST_CASE(test_simple_joinsplit_invalidity) CTransaction finalNewTx(newTx); BOOST_CHECK(finalNewTx.IsCoinBase()); } - BOOST_CHECK(!CheckTransaction(newTx, state)); + BOOST_CHECK(!CheckTransaction(newTx, state, verifier)); BOOST_CHECK(state.GetRejectReason() == "bad-cb-has-joinsplits"); } } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index d4966b186..ec6c55ee3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -14,6 +14,7 @@ #include "util.h" #include "utiltime.h" #include "wallet/wallet.h" +#include "zcash/Proof.hpp" #include #include @@ -411,7 +412,8 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CWalletTx wtx; ssValue >> wtx; CValidationState state; - if (!(CheckTransaction(wtx, state) && (wtx.GetHash() == hash) && state.IsValid())) + auto verifier = libzcash::ProofVerifier::Strict(); + if (!(CheckTransaction(wtx, state, verifier) && (wtx.GetHash() == hash) && state.IsValid())) return false; // Undo serialize changes in 31600 From 6f78f4f44c584c39501dd17814e289e32daa3cfa Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 10 Dec 2016 14:33:43 +1300 Subject: [PATCH 02/56] Only check cache validity for witnesses being incremented or decremented Fixes the bug resulting from #1904. --- src/wallet/wallet.cpp | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b424b8def..c49ca662d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -645,10 +645,14 @@ void CWallet::IncrementNoteWitnesses(const CBlockIndex* pindex, for (std::pair& wtxItem : mapWallet) { for (mapNoteData_t::value_type& item : wtxItem.second.mapNoteData) { CNoteData* nd = &(item.second); - // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); // Only increment witnesses that are behind the current height if (nd->witnessHeight < pindex->nHeight) { + // Check the validity of the cache + // The only time a note witnessed above the current height + // would be invalid here is during a reindex when blocks + // have been decremented, and we are incrementing the blocks + // immediately after. + assert(nWitnessCacheSize >= nd->witnesses.size()); // Witnesses being incremented should always be either -1 // (never incremented or decremented) or one below pindex assert((nd->witnessHeight == -1) || @@ -687,10 +691,11 @@ void CWallet::IncrementNoteWitnesses(const CBlockIndex* pindex, for (std::pair& wtxItem : mapWallet) { for (mapNoteData_t::value_type& item : wtxItem.second.mapNoteData) { CNoteData* nd = &(item.second); - // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); if (nd->witnessHeight < pindex->nHeight && nd->witnesses.size() > 0) { + // Check the validity of the cache + // See earlier comment about validity. + assert(nWitnessCacheSize >= nd->witnesses.size()); nd->witnesses.front().append(note_commitment); } } @@ -735,9 +740,10 @@ void CWallet::IncrementNoteWitnesses(const CBlockIndex* pindex, CNoteData* nd = &(item.second); if (nd->witnessHeight < pindex->nHeight) { nd->witnessHeight = pindex->nHeight; + // Check the validity of the cache + // See earlier comment about validity. + assert(nWitnessCacheSize >= nd->witnesses.size()); } - // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); } } @@ -754,10 +760,12 @@ void CWallet::DecrementNoteWitnesses(const CBlockIndex* pindex) for (std::pair& wtxItem : mapWallet) { for (mapNoteData_t::value_type& item : wtxItem.second.mapNoteData) { CNoteData* nd = &(item.second); - // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); // Only increment witnesses that are not above the current height if (nd->witnessHeight <= pindex->nHeight) { + // Check the validity of the cache + // See comment below (this would be invalid if there was a + // prior decrement). + assert(nWitnessCacheSize >= nd->witnesses.size()); // Witnesses being decremented should always be either -1 // (never incremented or decremented) or equal to pindex assert((nd->witnessHeight == -1) || @@ -776,7 +784,18 @@ void CWallet::DecrementNoteWitnesses(const CBlockIndex* pindex) for (mapNoteData_t::value_type& item : wtxItem.second.mapNoteData) { CNoteData* nd = &(item.second); // Check the validity of the cache - assert(nWitnessCacheSize >= nd->witnesses.size()); + // Technically if there are notes witnessed above the current + // height, their cache will now be invalid (relative to the new + // value of nWitnessCacheSize). However, this would only occur + // during a reindex, and by the time the reindex reaches the tip + // of the chain again, the existing witness caches will be valid + // again. + // We don't set nWitnessCacheSize to zero at the start of the + // reindex because the on-disk blocks had already resulted in a + // chain that didn't trigger the assertion below. + if (nd->witnessHeight < pindex->nHeight) { + assert(nWitnessCacheSize >= nd->witnesses.size()); + } } } // TODO: If nWitnessCache is zero, we need to regenerate the caches (#1302) From 6dac65b134ddd7fc4c2c33af683d04fe50bc267b Mon Sep 17 00:00:00 2001 From: Jay Graber Date: Mon, 12 Dec 2016 13:28:00 -0800 Subject: [PATCH 03/56] Update release process to check in with users who opened resolved issues --- doc/release-process.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/release-process.md b/doc/release-process.md index 93b97adaf..8f8851da7 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -112,8 +112,14 @@ Notify the Zcash DevOps engineer/sysadmin that the release has been tagged. They Then, verify that nodes can connect to the testnet server, and update the guide on the wiki to ensure the correct hostname is listed in the recommended zcash.conf. -## F. Update the Beta Guide +## F. Update the 1.0 User Guide + ## G. Publish the release announcement (blog, zcash-dev, slack) + +### G1. Check in with users who opened issues that were resolved in the release + +Contact all users who opened `user support` issues that were resolved in the release, and ask them if the release fixes or improves their issue. + ## H. Make and deploy deterministic builds - Run the [Gitian deterministic build environment](https://github.com/zcash/zcash-gitian) From c4fce3fc81380982c16d306c468e74d699633e3e Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Mon, 12 Dec 2016 19:14:42 -0700 Subject: [PATCH 04/56] Check that E' points are actually in G2 by ensuring they are of order r. --- src/gtest/test_proofs.cpp | 61 +++++++++++++++++++++++++++++++++++++++ src/zcash/Proof.cpp | 4 +++ 2 files changed, 65 insertions(+) diff --git a/src/gtest/test_proofs.cpp b/src/gtest/test_proofs.cpp index c6ffbf144..1e8d30bd7 100644 --- a/src/gtest/test_proofs.cpp +++ b/src/gtest/test_proofs.cpp @@ -22,6 +22,67 @@ typedef libsnark::default_r1cs_ppzksnark_pp::Fqe_type curve_Fq2; #include "version.h" #include "utilstrencodings.h" +TEST(proofs, g2_subgroup_check) +{ + // all G2 elements are order r + ASSERT_TRUE(libsnark::alt_bn128_modulus_r * curve_G2::random_element() == curve_G2::zero()); + + // but that doesn't mean all elements that satisfy the curve equation are in G2... + curve_G2 p = curve_G2::one(); + + while (1) { + // This will construct an order r(2q-r) point with high probability + p.X = curve_Fq2::random_element(); + try { + p.Y = ((p.X.squared() * p.X) + libsnark::alt_bn128_twist_coeff_b).sqrt(); + break; + } catch(...) {} + } + + ASSERT_TRUE(p.is_well_formed()); // it's on the curve + ASSERT_TRUE(libsnark::alt_bn128_modulus_r * p != curve_G2::zero()); // but not the order r subgroup.. + + { + // libsnark unfortunately doesn't check, and the pairing will complete + auto e = curve_Fr("149"); + auto a = curve_pp::reduced_pairing(curve_G1::one(), p); + auto b = curve_pp::reduced_pairing(e * curve_G1::one(), p); + + // though it will not preserve bilinearity + ASSERT_TRUE((a^e) != b); + } + + { + // so, our decompression API should not allow you to decompress G2 elements of that form! + CompressedG2 badp(p); + try { + auto newp = badp.to_libsnark_g2(); + FAIL() << "Expected std::runtime_error"; + } catch (std::runtime_error const & err) { + EXPECT_EQ(err.what(), std::string("point is not in G2")); + } catch(...) { + FAIL() << "Expected std::runtime_error"; + } + } + + // educational purposes: showing that E'(Fp2) is of order r(2q-r), + // by multiplying our random point in E' by (2q-r) = (q + q - r) to + // get an element in G2 + { + auto p1 = libsnark::alt_bn128_modulus_q * p; + p1 = p1 + p1; + p1 = p1 - (libsnark::alt_bn128_modulus_r * p); + + ASSERT_TRUE(p1.is_well_formed()); + ASSERT_TRUE(libsnark::alt_bn128_modulus_r * p1 == curve_G2::zero()); + + CompressedG2 goodp(p1); + auto newp = goodp.to_libsnark_g2(); + + ASSERT_TRUE(newp == p1); + } +} + TEST(proofs, sqrt_zero) { ASSERT_TRUE(curve_Fq::zero() == curve_Fq::zero().sqrt()); diff --git a/src/zcash/Proof.cpp b/src/zcash/Proof.cpp index 8a0ff2167..1b2199407 100644 --- a/src/zcash/Proof.cpp +++ b/src/zcash/Proof.cpp @@ -163,6 +163,10 @@ curve_G2 CompressedG2::to_libsnark_g2() const assert(r.is_well_formed()); + if (alt_bn128_modulus_r * r != curve_G2::zero()) { + throw std::runtime_error("point is not in G2"); + } + return r; } From 35ce717f96907d0ecf67c7673c3df2e27085cc27 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 13 Dec 2016 22:49:29 +1300 Subject: [PATCH 05/56] Fix bug in wallet tests --- src/wallet/gtest/test_wallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index c912a04c1..c97bae5de 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -51,7 +51,7 @@ public: void IncrementNoteWitnesses(const CBlockIndex* pindex, const CBlock* pblock, - ZCIncrementalMerkleTree tree) { + ZCIncrementalMerkleTree& tree) { CWallet::IncrementNoteWitnesses(pindex, pblock, tree); } void DecrementNoteWitnesses(const CBlockIndex* pindex) { From 0752d1f8c402ae87ce1b06c3a9b9de70e8cd8234 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 13 Dec 2016 22:53:11 +1300 Subject: [PATCH 06/56] Extract block-generation wallet test code into a function --- src/wallet/gtest/test_wallet.cpp | 128 +++++++++---------------------- 1 file changed, 36 insertions(+), 92 deletions(-) diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index c97bae5de..3a1215060 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -82,6 +82,28 @@ CWalletTx GetValidSpend(const libzcash::SpendingKey& sk, return GetValidSpend(*params, sk, note, value); } +JSOutPoint CreateValidBlock(TestWallet& wallet, + const libzcash::SpendingKey& sk, + const CBlockIndex& index, + CBlock& block, + ZCIncrementalMerkleTree& tree) { + auto wtx = GetValidReceive(sk, 50, true); + auto note = GetNote(sk, wtx, 0, 1); + auto nullifier = note.nullifier(sk); + + mapNoteData_t noteData; + JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; + CNoteData nd {sk.address(), nullifier}; + noteData[jsoutpt] = nd; + wtx.SetNoteData(noteData); + wallet.AddToWallet(wtx, true, NULL); + + block.vtx.push_back(wtx); + wallet.IncrementNoteWitnesses(&index, &block, tree); + + return jsoutpt; +} + TEST(wallet_tests, setup_datadir_location_run_as_first_test) { // Get temporary and unique path for file. boost::filesystem::path pathTemp = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path(); @@ -572,27 +594,14 @@ TEST(wallet_tests, cached_witnesses_chain_tip) { wallet.AddSpendingKey(sk); { - // First transaction (case tested in _empty_chain) - auto wtx = GetValidReceive(sk, 10, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); - - std::vector notes {jsoutpt}; - std::vector> witnesses; - // First block (case tested in _empty_chain) - block1.vtx.push_back(wtx); CBlockIndex index1(block1); index1.nHeight = 1; - wallet.IncrementNoteWitnesses(&index1, &block1, tree); + auto jsoutpt = CreateValidBlock(wallet, sk, index1, block1, tree); + // Called to fetch anchor + std::vector notes {jsoutpt}; + std::vector> witnesses; wallet.GetNoteWitnesses(notes, witnesses, anchor1); } @@ -667,47 +676,21 @@ TEST(wallet_tests, CachedWitnessesDecrementFirst) { wallet.AddSpendingKey(sk); { - // First transaction (case tested in _empty_chain) - auto wtx = GetValidReceive(sk, 10, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); - // First block (case tested in _empty_chain) CBlock block1; - block1.vtx.push_back(wtx); CBlockIndex index1(block1); index1.nHeight = 1; - wallet.IncrementNoteWitnesses(&index1, &block1, tree); + CreateValidBlock(wallet, sk, index1, block1, tree); } { - // Second transaction (case tested in _chain_tip) - auto wtx = GetValidReceive(sk, 50, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); + // Second block (case tested in _chain_tip) + index2.nHeight = 2; + auto jsoutpt = CreateValidBlock(wallet, sk, index2, block2, tree); + // Called to fetch anchor std::vector notes {jsoutpt}; std::vector> witnesses; - - // Second block (case tested in _chain_tip) - block2.vtx.push_back(wtx); - index2.nHeight = 2; - wallet.IncrementNoteWitnesses(&index2, &block2, tree); - // Called to fetch anchor wallet.GetNoteWitnesses(notes, witnesses, anchor2); } @@ -765,64 +748,25 @@ TEST(wallet_tests, CachedWitnessesCleanIndex) { wallet.AddSpendingKey(sk); { - // First transaction (case tested in _empty_chain) - auto wtx = GetValidReceive(sk, 10, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); - // First block (case tested in _empty_chain) - block1.vtx.push_back(wtx); index1.nHeight = 1; - wallet.IncrementNoteWitnesses(&index1, &block1, tree); + CreateValidBlock(wallet, sk, index1, block1, tree); } { - // Second transaction (case tested in _chain_tip) - auto wtx = GetValidReceive(sk, 50, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); - // Second block (case tested in _chain_tip) - block2.vtx.push_back(wtx); index2.nHeight = 2; - wallet.IncrementNoteWitnesses(&index2, &block2, tree); + CreateValidBlock(wallet, sk, index2, block2, tree); } { - // Third transaction - auto wtx = GetValidReceive(sk, 20, true); - auto note = GetNote(sk, wtx, 0, 1); - auto nullifier = note.nullifier(sk); - - mapNoteData_t noteData; - JSOutPoint jsoutpt {wtx.GetHash(), 0, 1}; - CNoteData nd {sk.address(), nullifier}; - noteData[jsoutpt] = nd; - wtx.SetNoteData(noteData); - wallet.AddToWallet(wtx, true, NULL); + // Third block + index3.nHeight = 3; + auto jsoutpt = CreateValidBlock(wallet, sk, index3, block3, tree); std::vector notes {jsoutpt}; std::vector> witnesses; uint256 anchor3; - - // Third block - block3.vtx.push_back(wtx); - index3.nHeight = 3; - wallet.IncrementNoteWitnesses(&index3, &block3, tree); wallet.GetNoteWitnesses(notes, witnesses, anchor3); // Now pretend we are reindexing: the chain is cleared, and each block is From 78f4e0ef373ff05c598102034460b96a74048097 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 13 Dec 2016 22:53:29 +1300 Subject: [PATCH 07/56] Rewrite reindex test to check beyond the max witness cache size --- src/wallet/gtest/test_wallet.cpp | 122 +++++++++++++++---------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index 3a1215060..9bcc5f533 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -736,77 +736,77 @@ TEST(wallet_tests, CachedWitnessesDecrementFirst) { TEST(wallet_tests, CachedWitnessesCleanIndex) { TestWallet wallet; - CBlock block1; - CBlock block2; - CBlock block3; - CBlockIndex index1(block1); - CBlockIndex index2(block2); - CBlockIndex index3(block3); + std::vector blocks; + std::vector indices; + std::vector notes; + std::vector anchors; ZCIncrementalMerkleTree tree; + ZCIncrementalMerkleTree riTree = tree; + std::vector> witnesses; auto sk = libzcash::SpendingKey::random(); wallet.AddSpendingKey(sk); - { - // First block (case tested in _empty_chain) - index1.nHeight = 1; - CreateValidBlock(wallet, sk, index1, block1, tree); + // Generate a chain + size_t numBlocks = WITNESS_CACHE_SIZE + 10; + blocks.resize(numBlocks); + indices.resize(numBlocks); + for (size_t i = 0; i < numBlocks; i++) { + indices[i].nHeight = i; + auto old = tree.root(); + auto jsoutpt = CreateValidBlock(wallet, sk, indices[i], blocks[i], tree); + EXPECT_NE(old, tree.root()); + notes.push_back(jsoutpt); + + witnesses.clear(); + uint256 anchor; + wallet.GetNoteWitnesses(notes, witnesses, anchor); + for (size_t j = 0; j <= i; j++) { + EXPECT_TRUE((bool) witnesses[j]); + } + anchors.push_back(anchor); } - { - // Second block (case tested in _chain_tip) - index2.nHeight = 2; - CreateValidBlock(wallet, sk, index2, block2, tree); - } - - { - // Third block - index3.nHeight = 3; - auto jsoutpt = CreateValidBlock(wallet, sk, index3, block3, tree); - - std::vector notes {jsoutpt}; - std::vector> witnesses; - uint256 anchor3; - wallet.GetNoteWitnesses(notes, witnesses, anchor3); - - // Now pretend we are reindexing: the chain is cleared, and each block is - // used to increment witnesses again. - wallet.IncrementNoteWitnesses(&index1, &block1, tree); - uint256 anchor3a; + // Now pretend we are reindexing: the chain is cleared, and each block is + // used to increment witnesses again. + for (size_t i = 0; i < numBlocks; i++) { + ZCIncrementalMerkleTree riPrevTree {riTree}; + wallet.IncrementNoteWitnesses(&(indices[i]), &(blocks[i]), riTree); witnesses.clear(); - wallet.GetNoteWitnesses(notes, witnesses, anchor3a); - EXPECT_TRUE((bool) witnesses[0]); - // Should equal third anchor because witness cache unaffected - EXPECT_EQ(anchor3, anchor3a); + uint256 anchor; + wallet.GetNoteWitnesses(notes, witnesses, anchor); + for (size_t j = 0; j < numBlocks; j++) { + EXPECT_TRUE((bool) witnesses[j]); + } + // Should equal final anchor because witness cache unaffected + EXPECT_EQ(anchors.back(), anchor); - wallet.IncrementNoteWitnesses(&index2, &block2, tree); - uint256 anchor3b; - witnesses.clear(); - wallet.GetNoteWitnesses(notes, witnesses, anchor3b); - EXPECT_TRUE((bool) witnesses[0]); - EXPECT_EQ(anchor3, anchor3b); + if ((i == 5) || (i == 50)) { + // Pretend a reorg happened that was recorded in the block files + { + wallet.DecrementNoteWitnesses(&(indices[i])); + witnesses.clear(); + uint256 anchor; + wallet.GetNoteWitnesses(notes, witnesses, anchor); + for (size_t j = 0; j < numBlocks; j++) { + EXPECT_TRUE((bool) witnesses[j]); + } + // Should equal final anchor because witness cache unaffected + EXPECT_EQ(anchors.back(), anchor); + } - // Pretend a reorg happened that was recorded in the block files - wallet.DecrementNoteWitnesses(&index2); - uint256 anchor3c; - witnesses.clear(); - wallet.GetNoteWitnesses(notes, witnesses, anchor3c); - EXPECT_TRUE((bool) witnesses[0]); - EXPECT_EQ(anchor3, anchor3c); - - wallet.IncrementNoteWitnesses(&index2, &block2, tree); - uint256 anchor3d; - witnesses.clear(); - wallet.GetNoteWitnesses(notes, witnesses, anchor3d); - EXPECT_TRUE((bool) witnesses[0]); - EXPECT_EQ(anchor3, anchor3d); - - wallet.IncrementNoteWitnesses(&index3, &block3, tree); - uint256 anchor3e; - witnesses.clear(); - wallet.GetNoteWitnesses(notes, witnesses, anchor3e); - EXPECT_TRUE((bool) witnesses[0]); - EXPECT_EQ(anchor3, anchor3e); + { + wallet.IncrementNoteWitnesses(&(indices[i]), &(blocks[i]), riPrevTree); + witnesses.clear(); + uint256 anchor; + wallet.GetNoteWitnesses(notes, witnesses, anchor); + for (size_t j = 0; j < numBlocks; j++) { + EXPECT_TRUE((bool) witnesses[j]); + } + // Should equal final anchor because witness cache unaffected + EXPECT_EQ(anchors.back(), anchor); + } + } } } From c0ec0e756bb568096141752d956f89e6fabfdf13 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 13 Dec 2016 22:53:52 +1300 Subject: [PATCH 08/56] Fix bug in IncrementNoteWitness() --- src/wallet/wallet.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c49ca662d..8c32dcb1a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -704,7 +704,8 @@ void CWallet::IncrementNoteWitnesses(const CBlockIndex* pindex, // If this is our note, witness it if (txIsOurs) { JSOutPoint jsoutpt {hash, i, j}; - if (mapWallet[hash].mapNoteData.count(jsoutpt)) { + if (mapWallet[hash].mapNoteData.count(jsoutpt) && + mapWallet[hash].mapNoteData[jsoutpt].witnessHeight < pindex->nHeight) { CNoteData* nd = &(mapWallet[hash].mapNoteData[jsoutpt]); if (nd->witnesses.size() > 0) { // We think this can happen because we write out the From 4082dcb10ff976ce9d850e558f30e23cd7477272 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 14 Dec 2016 15:26:02 +1300 Subject: [PATCH 09/56] Extend createjoinsplit to benchmark parallel JoinSplits Closes #1940 --- qa/zcash/performance-measurements.sh | 6 +++--- src/wallet/rpcwallet.cpp | 10 +++++++++- src/zcbenchmarks.cpp | 21 +++++++++++++++++++++ src/zcbenchmarks.h | 1 + 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/qa/zcash/performance-measurements.sh b/qa/zcash/performance-measurements.sh index 851adef09..c210a8e88 100755 --- a/qa/zcash/performance-measurements.sh +++ b/qa/zcash/performance-measurements.sh @@ -74,7 +74,7 @@ case "$1" in zcash_rpc zcbenchmark parameterloading 10 ;; createjoinsplit) - zcash_rpc zcbenchmark createjoinsplit 10 + zcash_rpc zcbenchmark createjoinsplit 10 "${@:3}" ;; verifyjoinsplit) zcash_rpc zcbenchmark verifyjoinsplit 1000 "\"$RAWJOINSPLIT\"" @@ -111,7 +111,7 @@ case "$1" in zcash_rpc zcbenchmark parameterloading 1 ;; createjoinsplit) - zcash_rpc zcbenchmark createjoinsplit 1 + zcash_rpc zcbenchmark createjoinsplit 1 "${@:3}" ;; verifyjoinsplit) zcash_rpc zcbenchmark verifyjoinsplit 1 "\"$RAWJOINSPLIT\"" @@ -146,7 +146,7 @@ case "$1" in zcash_rpc zcbenchmark parameterloading 1 ;; createjoinsplit) - zcash_rpc zcbenchmark createjoinsplit 1 + zcash_rpc zcbenchmark createjoinsplit 1 "${@:3}" ;; verifyjoinsplit) zcash_rpc zcbenchmark verifyjoinsplit 1 "\"$RAWJOINSPLIT\"" diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..161061fbb 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2465,7 +2465,15 @@ Value zc_benchmark(const json_spirit::Array& params, bool fHelp) } else if (benchmarktype == "parameterloading") { sample_times.push_back(benchmark_parameter_loading()); } else if (benchmarktype == "createjoinsplit") { - sample_times.push_back(benchmark_create_joinsplit()); + if (params.size() < 3) { + sample_times.push_back(benchmark_create_joinsplit()); + } else { + int nThreads = params[2].get_int(); + std::vector vals = benchmark_create_joinsplit_threaded(nThreads); + // Divide by nThreads^2 to get average seconds per JoinSplit because + // we are running one JoinSplit per thread. + sample_times.push_back(std::accumulate(vals.begin(), vals.end(), 0.0) / (nThreads*nThreads)); + } } else if (benchmarktype == "verifyjoinsplit") { sample_times.push_back(benchmark_verify_joinsplit(samplejoinsplit)); } else if (benchmarktype == "solveequihash") { diff --git a/src/zcbenchmarks.cpp b/src/zcbenchmarks.cpp index b88ebb9e1..e6f8440ac 100644 --- a/src/zcbenchmarks.cpp +++ b/src/zcbenchmarks.cpp @@ -95,6 +95,27 @@ double benchmark_create_joinsplit() return ret; } +std::vector benchmark_create_joinsplit_threaded(int nThreads) +{ + std::vector ret; + std::vector> tasks; + std::vector threads; + for (int i = 0; i < nThreads; i++) { + std::packaged_task task(&benchmark_create_joinsplit); + tasks.emplace_back(task.get_future()); + threads.emplace_back(std::move(task)); + } + std::future_status status; + for (auto it = tasks.begin(); it != tasks.end(); it++) { + it->wait(); + ret.push_back(it->get()); + } + for (auto it = threads.begin(); it != threads.end(); it++) { + it->join(); + } + return ret; +} + double benchmark_verify_joinsplit(const JSDescription &joinsplit) { struct timeval tv_start; diff --git a/src/zcbenchmarks.h b/src/zcbenchmarks.h index 318921002..b2bc2e373 100644 --- a/src/zcbenchmarks.h +++ b/src/zcbenchmarks.h @@ -7,6 +7,7 @@ extern double benchmark_sleep(); extern double benchmark_parameter_loading(); extern double benchmark_create_joinsplit(); +extern std::vector benchmark_create_joinsplit_threaded(int nThreads); extern double benchmark_solve_equihash(); extern std::vector benchmark_solve_equihash_threaded(int nThreads); extern double benchmark_verify_joinsplit(const JSDescription &joinsplit); From 41c616214c45b46fcf260fb4a260c0eb65240d30 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 15 Dec 2016 14:30:40 +1300 Subject: [PATCH 10/56] Update payment API docs to recommend -rescan for fixing witness errors --- doc/payment-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/payment-api.md b/doc/payment-api.md index ccbb5725c..3560add64 100644 --- a/doc/payment-api.md +++ b/doc/payment-api.md @@ -151,7 +151,7 @@ RPC_WALLET_ERROR (-4) | _Unspecified problem with wallet_ ----------------------| ------------------------------------- "Could not find previous JoinSplit anchor" | Try restarting node with `-reindex`. "Error decrypting output note of previous JoinSplit: __" | -"Could not find witness for note commitment" | Try restarting node with `-reindex`. +"Could not find witness for note commitment" | Try restarting node with `-rescan`. "Witness for note commitment is null" | Missing witness for note commitement. "Witness for spendable note does not have same anchor as change input" | Invalid anchor for spendable note witness. "Not enough funds to pay miners fee" | Retry with sufficient funds. From 5d6e1aa60f4837f0f8edae047942ecdf36d505f7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 15 Dec 2016 15:50:26 +1300 Subject: [PATCH 11/56] Add total number of commitments to getblockchaininfo --- src/rpcblockchain.cpp | 5 +++++ src/zcash/IncrementalMerkleTree.cpp | 19 +++++++++++++++++++ src/zcash/IncrementalMerkleTree.hpp | 2 ++ 3 files changed, 26 insertions(+) diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 4c92979b2..4e6d42a6b 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -533,6 +533,7 @@ Value getblockchaininfo(const Array& params, bool fHelp) " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" + " \"commitments\": xxxxxx, (numeric) the current number of note commitments in the commitment tree\n" " \"softforks\": [ (array) status of softforks in progress\n" " {\n" " \"id\": \"xxxx\", (string) name of softfork\n" @@ -564,6 +565,10 @@ Value getblockchaininfo(const Array& params, bool fHelp) obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); obj.push_back(Pair("pruned", fPruneMode)); + ZCIncrementalMerkleTree tree; + pcoinsTip->GetAnchorAt(pcoinsTip->GetBestAnchor(), tree); + obj.push_back(Pair("commitments", tree.size())); + const Consensus::Params& consensusParams = Params().GetConsensus(); CBlockIndex* tip = chainActive.Tip(); Array softforks; diff --git a/src/zcash/IncrementalMerkleTree.cpp b/src/zcash/IncrementalMerkleTree.cpp index cf2d00af7..d59e707fa 100644 --- a/src/zcash/IncrementalMerkleTree.cpp +++ b/src/zcash/IncrementalMerkleTree.cpp @@ -81,6 +81,25 @@ Hash IncrementalMerkleTree::last() const { } } +template +size_t IncrementalMerkleTree::size() const { + size_t ret = 0; + if (left) { + ret++; + } + if (right) { + ret++; + } + // Treat occupation of parents array as a binary number + // (right-shifted by 1) + for (size_t i = 0; i < parents.size(); i++) { + if (parents[i]) { + ret += (1 << (i+1)); + } + } + return ret; +} + template void IncrementalMerkleTree::append(Hash obj) { if (is_complete(Depth)) { diff --git a/src/zcash/IncrementalMerkleTree.hpp b/src/zcash/IncrementalMerkleTree.hpp index 6c50192c8..67b356318 100644 --- a/src/zcash/IncrementalMerkleTree.hpp +++ b/src/zcash/IncrementalMerkleTree.hpp @@ -75,6 +75,8 @@ public: parents.size() * 32; // parents } + size_t size() const; + void append(Hash obj); Hash root() const { return root(Depth, std::deque()); From 83ce8a4d7cde8e89628e8e53f924633582ba1d16 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 15 Dec 2016 16:33:09 +1300 Subject: [PATCH 12/56] Update version to 1.0.4 --- README.md | 2 +- configure.ac | 2 +- contrib/DEBIAN/control | 2 +- contrib/gitian-descriptors/gitian-linux.yml | 2 +- src/clientversion.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 346a037ab..c15d296e3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Zcash 1.0.3 +Zcash 1.0.4 =========== What is Zcash? diff --git a/configure.ac b/configure.ac index 3a6ce2754..a6e40ab5a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 1) define(_CLIENT_VERSION_MINOR, 0) -define(_CLIENT_VERSION_REVISION, 3) +define(_CLIENT_VERSION_REVISION, 4) define(_CLIENT_VERSION_BUILD, 50) define(_ZC_BUILD_VAL, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, m4_incr(_CLIENT_VERSION_BUILD), m4_eval(_CLIENT_VERSION_BUILD < 50), 1, m4_eval(_CLIENT_VERSION_BUILD - 24), m4_eval(_CLIENT_VERSION_BUILD == 50), 1, , m4_eval(_CLIENT_VERSION_BUILD - 50))) define(_CLIENT_VERSION_SUFFIX, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, _CLIENT_VERSION_REVISION-beta$1, m4_eval(_CLIENT_VERSION_BUILD < 50), 1, _CLIENT_VERSION_REVISION-rc$1, m4_eval(_CLIENT_VERSION_BUILD == 50), 1, _CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION-$1))) diff --git a/contrib/DEBIAN/control b/contrib/DEBIAN/control index 4046e372b..b24ffc54d 100644 --- a/contrib/DEBIAN/control +++ b/contrib/DEBIAN/control @@ -10,7 +10,7 @@ Build-Depends: autoconf, automake, bsdmainutils, build-essential Vcs-Git: https://github.com/zcash/zcash.git Vcs-Browser: https://github.com/zcash/zcash Package: zcash -Version: 1.0.3 +Version: 1.0.4 Architecture: amd64 Depends: libgomp1 Description: An implementation of the "Zerocash" protocol. diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 6b83346a7..49c27537d 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "zcash-1.0.3" +name: "zcash-1.0.4" enable_cache: true distro: "debian" suites: diff --git a/src/clientversion.h b/src/clientversion.h index 41de19837..6ca038285 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -16,7 +16,7 @@ //! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it #define CLIENT_VERSION_MAJOR 1 #define CLIENT_VERSION_MINOR 0 -#define CLIENT_VERSION_REVISION 3 +#define CLIENT_VERSION_REVISION 4 #define CLIENT_VERSION_BUILD 50 //! Set to true for release, false for prerelease or test build From 196df30d72a6ec959ff156221f36fc2edb3023f0 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 15 Dec 2016 16:45:18 +1300 Subject: [PATCH 13/56] Update man pages --- contrib/DEBIAN/manpages/zcash-cli.1 | 4 +-- contrib/DEBIAN/manpages/zcashd.1 | 40 +++++++++++++++++++---------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/contrib/DEBIAN/manpages/zcash-cli.1 b/contrib/DEBIAN/manpages/zcash-cli.1 index 01e778a80..e5b47babd 100644 --- a/contrib/DEBIAN/manpages/zcash-cli.1 +++ b/contrib/DEBIAN/manpages/zcash-cli.1 @@ -1,9 +1,9 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH ZCASH-CLI "1" "November 2016" "Zcash RPC client version v1.0.3" "User Commands" +.TH ZCASH-CLI "1" "December 2016" "Zcash RPC client version v1.0.4" "User Commands" .SH NAME zcash-cli \- RPC client for the Zcash daemon .SH DESCRIPTION -Zcash RPC client version v1.0.3 +Zcash RPC client version v1.0.4 .SS "Usage:" .TP zcash\-cli [options] [params] diff --git a/contrib/DEBIAN/manpages/zcashd.1 b/contrib/DEBIAN/manpages/zcashd.1 index 9a0362630..a110108be 100644 --- a/contrib/DEBIAN/manpages/zcashd.1 +++ b/contrib/DEBIAN/manpages/zcashd.1 @@ -1,9 +1,9 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH ZCASHD "1" "November 2016" "Zcash Daemon version v1.0.3" "User Commands" +.TH ZCASHD "1" "December 2016" "Zcash Daemon version v1.0.4" "User Commands" .SH NAME zcashd \- Network daemon for interacting with the Zcash blockchain .SH DESCRIPTION -Zcash Daemon version v1.0.3 +Zcash Daemon version v1.0.4 .SS "Usage:" .TP zcashd [options] @@ -62,7 +62,7 @@ Keep at most unconnectable transactions in memory (default: 100) .HP \fB\-par=\fR .IP -Set the number of script verification threads (\fB\-8\fR to 16, 0 = auto, <0 = +Set the number of script verification threads (\fB\-4\fR to 16, 0 = auto, <0 = leave that many cores free, default: 0) .HP \fB\-pid=\fR @@ -219,7 +219,7 @@ Fee (in BTC/kB) to add to transactions you send (default: 0.00) .HP \fB\-rescan\fR .IP -Rescan the block chain for missing wallet transactions on startup +Rescan the blockchain for missing wallet transactions on startup .HP \fB\-salvagewallet\fR .IP @@ -271,9 +271,9 @@ Debugging/Testing options: \fB\-debug=\fR .IP Output debugging information (default: 0, supplying is -optional). If is not supplied, output all debugging -information. can be: addrman, alert, bench, coindb, db, lock, -rand, rpc, selectcoins, mempool, net, proxy, prune. +optional). If is not supplied or if = 1, output +all debugging information. can be: addrman, alert, bench, +coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune. .HP \fB\-gen\fR .IP @@ -303,7 +303,7 @@ Prepend debug output with timestamp (default: 1) \fB\-minrelaytxfee=\fR .IP Fees (in BTC/Kb) smaller than this are considered zero fee for relaying -(default: 0.00005) +(default: 0.00001) .HP \fB\-printtoconsole\fR .IP @@ -336,12 +336,12 @@ Set minimum block size in bytes (default: 0) .HP \fB\-blockmaxsize=\fR .IP -Set maximum block size in bytes (default: 750000) +Set maximum block size in bytes (default: 2000000) .HP \fB\-blockprioritysize=\fR .IP Set maximum size of high\-priority/low\-fee transactions in bytes -(default: 50000) +(default: 1000000) .PP RPC server options: .HP @@ -382,10 +382,6 @@ multiple times \fB\-rpcthreads=\fR .IP Set the number of threads to service RPC calls (default: 4) -.HP -\fB\-rpckeepalive\fR -.IP -RPC support for HTTP persistent connections (default: 1) .PP RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions) .HP @@ -405,6 +401,22 @@ Server private key (default: server.pem) .IP Acceptable ciphers (default: TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH) +.PP +Metrics Options (only if \fB\-daemon\fR and \fB\-printtoconsole\fR are not set): +.HP +\fB\-showmetrics\fR +.IP +Show metrics on stdout (default: 1 if running in a console, 0 otherwise) +.HP +\fB\-metricsui\fR +.IP +Set to 1 for a persistent metrics screen, 0 for sequential metrics +output (default: 1 if running in a console, 0 otherwise) +.HP +\fB\-metricsrefreshtime\fR +.IP +Number of seconds between metrics refreshes (default: 1 if running in a +console, 600 otherwise) .SH COPYRIGHT Copyright \(co 2009\-2016 The Bitcoin Core Developers .br From 611f25b6044e92d146e6272539f0523256a9cacf Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 15 Dec 2016 16:47:49 +1300 Subject: [PATCH 14/56] Release notes, authors, changelog --- contrib/DEBIAN/changelog | 6 ++ doc/authors.md | 13 ++-- doc/release-notes/release-notes-1.0.4.md | 75 ++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 doc/release-notes/release-notes-1.0.4.md diff --git a/contrib/DEBIAN/changelog b/contrib/DEBIAN/changelog index 3b3305c93..eb9c5f415 100644 --- a/contrib/DEBIAN/changelog +++ b/contrib/DEBIAN/changelog @@ -1,3 +1,9 @@ +zcash (1.0.4) jessie; urgency=medium + + * 1.0.4 release. + + -- Zcash Company Thu, 15 Dec 2016 16:46:14 +1300 + zcash (1.0.3) jessie; urgency=medium * 1.0.3 release. diff --git a/doc/authors.md b/doc/authors.md index c13e6a499..2933964b5 100644 --- a/doc/authors.md +++ b/doc/authors.md @@ -1,13 +1,13 @@ Zcash Contributors ================== -Jack Grigg (269) -Simon Liu (199) -Sean Bowe (168) +Jack Grigg (301) +Simon Liu (207) +Sean Bowe (176) Taylor Hornby (65) Daira Hopwood (62) Kevin Gallagher (38) -Jay Graber (32) +Jay Graber (34) Nathan Wilcox (10) Wladimir J. van der Laan (9) Pieter Wuille (8) @@ -15,6 +15,7 @@ Cory Fields (7) ITH4Coinomia (4) David Mercer (4) 4ZEC (4) +lpescher (3) Patrick Strateman (3) Paige Peterson (3) MarcoFalke (3) @@ -27,13 +28,17 @@ kazcw (1) fanquake (1) ayleph (1) Tom Ritter (1) +Scott (1) S. Matthew English (1) Philip Kaufmann (1) Louis Nyffenegger (1) Lars-Magnus Skog (1) +Jeffrey Walton (1) Gaurav Rana (1) Ethan Heilman (1) +Christian von Roques (1) Chirag Davé (1) Cameron Boehmer (1) Bryan Stitt (1) +Bitcoin Error Log (1) Alex (1) diff --git a/doc/release-notes/release-notes-1.0.4.md b/doc/release-notes/release-notes-1.0.4.md new file mode 100644 index 000000000..1cb73c393 --- /dev/null +++ b/doc/release-notes/release-notes-1.0.4.md @@ -0,0 +1,75 @@ +Bitcoin Error Log (1): + Edit for grammar: "block chain" + +Christian von Roques (1): + bash-completion: Adapt for 0.12 and 0.13 + +Jack Grigg (32): + Add getlocalsolps and getnetworksolps RPC calls, show them in getmininginfo + Add benchmark for attempting decryption of notes + Add benchmark for incrementing note witnesses + Add -metricsui flag to toggle between persistent screen and rolling metrics + Add -metricsrefreshtime option + Only show metrics by default if stdout is a TTY + Document metrics screen options + Clarify that metrics options are only useful without -daemon and -printtoconsole + Increase length of metrics divider + Write witness caches when writing the best block + Apply miniupnpc patches to enable compilation on Solaris 11 + Add an upstream miniupnpc patch revision + Address review comments, tweak strings + Change function names to not clash with Bitcoin, apply to correct binaries + Add bash completion files to Debian package + Always bash-complete the default account + Add Zcash RPC commands to CLI argument completion + Document behaviour of CWallet::SetBestChain + Fix indentation + Generate JS for trydecryptnotes, make number of addresses a variable + Add JS to second block to ensure witnesses are incremented + Skip JoinSplit verification before the last checkpoint + Add a reindex test that fails because of a bug in decrementing witness caches + Make the test pass by fixing the bug! + Only check cache validity for witnesses being incremented or decremented + Fix bug in wallet tests + Extract block-generation wallet test code into a function + Rewrite reindex test to check beyond the max witness cache size + Fix bug in IncrementNoteWitness() + Update payment API docs to recommend -rescan for fixing witness errors + Update version to 1.0.4 + Update man pages + +Jay Graber (2): + Replace bitcoin with zcash in rpcprotocol.cpp + Gather release notes from previous release to HEAD + +Jeffrey Walton (1): + Add porter dev overrides for CC, CXX, MAKE, BUILD, HOST + +Scott (1): + Metrics - Don't exclaim unless > 1 + +Sean Bowe (8): + Isolate verification to a `ProofVerifier` context object that allows verification behavior to be tuned by the caller. + Regression test. + Ensure cache contains valid entry when anchor is popped. + Ensure ProofVerifier cannot be accidentally copied. + Rename Dummy to Disabled. + Add more tests for ProofVerifier. + ASSERT_TRUE -> ASSERT_FALSE + Check that E' points are actually in G2 by ensuring they are of order r. + +Simon Liu (8): + Fix stale comment referencing upstream block interval + Add checkpoint at block height 15000 + Closes #1857. Fixes bug where tx spending only notes had priority of 0. + Closes #1901. Increase default settings for the max block size when mining and the amount of space available for priority transactions. + Closes #1903. Add fee parameter to z_sendmany. + Fixes #1823. Witness anchors for input notes no longer cross block boundaries. + Increase timeout as laptops on battery power have cpu throttling. + WitnessAnchorData only needs to store one witness per JSOutPoint. + +lpescher (3): + Make command line option to show all debugging consistent with similar options + Update documentation to match the #4219 change + Update help message to match the #4219 change + From f3e4968626c37b5bd90ad29fc0d9b9bacd838466 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 16 Dec 2016 10:44:04 +1300 Subject: [PATCH 15/56] Only enable getblocktemplate when wallet is enabled --- src/rpcmining.cpp | 2 ++ src/rpcserver.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index d58e70adc..474541bd4 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -401,6 +401,7 @@ static Value BIP22ValidationResult(const CValidationState& state) return "valid?"; } +#ifdef ENABLE_WALLET Value getblocktemplate(const Array& params, bool fHelp) { if (fHelp || params.size() > 1) @@ -694,6 +695,7 @@ Value getblocktemplate(const Array& params, bool fHelp) return result; } +#endif class submitblock_StateCatcher : public CValidationInterface { diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index e354e91a4..00ea417c3 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -302,7 +302,9 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifychain", &verifychain, true }, /* Mining */ +#ifdef ENABLE_WALLET { "mining", "getblocktemplate", &getblocktemplate, true }, +#endif { "mining", "getmininginfo", &getmininginfo, true }, { "mining", "getlocalsolps", &getlocalsolps, true }, { "mining", "getnetworksolps", &getnetworksolps, true }, From 6daab5b4fcb409812275405006b38676a1439180 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 16 Dec 2016 10:52:00 +1300 Subject: [PATCH 16/56] Only run wallet tests when wallet is enabled --- src/Makefile.gtest.include | 4 ++-- src/gtest/test_keystore.cpp | 4 ++++ src/{ => wallet}/gtest/test_wallet_zkeys.cpp | 0 3 files changed, 6 insertions(+), 2 deletions(-) rename src/{ => wallet}/gtest/test_wallet_zkeys.cpp (100%) diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index a7071f3b7..1c9203ffa 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -11,7 +11,6 @@ zcash_gtest_SOURCES = \ gtest/json_test_vectors.cpp \ gtest/json_test_vectors.h \ gtest/test_foundersreward.cpp \ - gtest/test_wallet_zkeys.cpp \ gtest/test_jsonspirit.cpp \ gtest/test_tautology.cpp \ gtest/test_equihash.cpp \ @@ -31,7 +30,8 @@ zcash_gtest_SOURCES = \ gtest/test_checkblock.cpp if ENABLE_WALLET zcash_gtest_SOURCES += \ - wallet/gtest/test_wallet.cpp + wallet/gtest/test_wallet.cpp \ + wallet/gtest/test_wallet_zkeys.cpp endif zcash_gtest_CPPFLAGS = -DMULTICORE -fopenmp -DBINARY_OUTPUT -DCURVE_ALT_BN128 -DSTATIC diff --git a/src/gtest/test_keystore.cpp b/src/gtest/test_keystore.cpp index 26fde42d9..e94aea53e 100644 --- a/src/gtest/test_keystore.cpp +++ b/src/gtest/test_keystore.cpp @@ -2,7 +2,9 @@ #include "keystore.h" #include "random.h" +#ifdef ENABLE_WALLET #include "wallet/crypter.h" +#endif #include "zcash/Address.hpp" TEST(keystore_tests, store_and_retrieve_spending_key) { @@ -44,6 +46,7 @@ TEST(keystore_tests, store_and_retrieve_note_decryptor) { EXPECT_EQ(ZCNoteDecryption(sk.viewing_key()), decOut); } +#ifdef ENABLE_WALLET class TestCCryptoKeyStore : public CCryptoKeyStore { public: @@ -125,3 +128,4 @@ TEST(keystore_tests, store_and_retrieve_spending_key_in_encrypted_store) { ASSERT_EQ(1, addrs.count(addr)); ASSERT_EQ(1, addrs.count(addr2)); } +#endif diff --git a/src/gtest/test_wallet_zkeys.cpp b/src/wallet/gtest/test_wallet_zkeys.cpp similarity index 100% rename from src/gtest/test_wallet_zkeys.cpp rename to src/wallet/gtest/test_wallet_zkeys.cpp From d0ce704abefb2e6d524401ca08e85882195f6e18 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 17 Dec 2016 02:39:35 +1300 Subject: [PATCH 17/56] Add a tool for profiling the creation of JoinSplits --- src/Makefile.zcash.include | 13 ++++++++++++- src/zcash/CreateJoinSplit.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 src/zcash/CreateJoinSplit.cpp diff --git a/src/Makefile.zcash.include b/src/Makefile.zcash.include index 341ba2736..59aeb861f 100644 --- a/src/Makefile.zcash.include +++ b/src/Makefile.zcash.include @@ -1,5 +1,6 @@ bin_PROGRAMS += \ - zcash/GenerateParams + zcash/GenerateParams \ + zcash/CreateJoinSplit # tool for generating our public parameters zcash_GenerateParams_SOURCES = zcash/GenerateParams.cpp @@ -9,3 +10,13 @@ zcash_GenerateParams_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ $(LIBZCASH_LIBS) + +# tool for profiling the creation of joinsplits +zcash_CreateJoinSplit_SOURCES = zcash/CreateJoinSplit.cpp +zcash_CreateJoinSplit_LDADD = \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBZCASH) \ + $(BOOST_LIBS) \ + $(LIBZCASH_LIBS) diff --git a/src/zcash/CreateJoinSplit.cpp b/src/zcash/CreateJoinSplit.cpp new file mode 100644 index 000000000..65dc7f65c --- /dev/null +++ b/src/zcash/CreateJoinSplit.cpp @@ -0,0 +1,32 @@ +// Copyright (c) 2016 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "../util.h" +#include "primitives/transaction.h" +#include "zcash/JoinSplit.hpp" + +using namespace libzcash; + +int main(int argc, char **argv) +{ + auto p = ZCJoinSplit::Unopened(); + p->loadVerifyingKey((ZC_GetParamsDir() / "sprout-verifying.key").string()); + p->setProvingKeyPath((ZC_GetParamsDir() / "sprout-proving.key").string()); + p->loadProvingKey(); + + // construct a proof. + + for (int i = 0; i < 5; i++) { + uint256 anchor = ZCIncrementalMerkleTree().root(); + uint256 pubKeyHash; + + JSDescription jsdesc(*p, + pubKeyHash, + anchor, + {JSInput(), JSInput()}, + {JSOutput(), JSOutput()}, + 0, + 0); + } +} From fc538ec2ea3e426dcfa69a9426eddf359716b49b Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Fri, 16 Dec 2016 13:50:55 -0700 Subject: [PATCH 18/56] Add test for IncrementalMerkleTree::size(). --- src/gtest/test_merkletree.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/gtest/test_merkletree.cpp b/src/gtest/test_merkletree.cpp index efe5587e5..cf1d8617a 100644 --- a/src/gtest/test_merkletree.cpp +++ b/src/gtest/test_merkletree.cpp @@ -79,6 +79,9 @@ void test_tree( // The tree doesn't have a 'last' element added since it's blank. ASSERT_THROW(tree.last(), std::runtime_error); + // The tree is empty. + ASSERT_TRUE(tree.size() == 0); + // We need to witness at every single point in the tree, so // that the consistency of the tree and the merkle paths can // be checked. @@ -93,6 +96,9 @@ void test_tree( // Now append a commitment to the tree tree.append(test_commitment); + // Size incremented by one. + ASSERT_TRUE(tree.size() == i+1); + // Last element added to the tree was `test_commitment` ASSERT_TRUE(tree.last() == test_commitment); From 8e0fc390971aabcef5dd868eee5370d4a36eda65 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Jan 2017 11:23:21 +0100 Subject: [PATCH 19/56] Exclude test binaries from make install Closes #1943. --- src/Makefile.am | 1 + src/Makefile.gtest.include | 2 +- src/Makefile.test.include | 2 +- src/Makefile.zcash.include | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index e37510ae4..78684a4eb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -59,6 +59,7 @@ LIBZCASH_CONSENSUS= endif bin_PROGRAMS = +noinst_PROGRAMS = TESTS = if BUILD_BITCOIND diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index a7071f3b7..2ae1d64c6 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -1,5 +1,5 @@ TESTS += zcash-gtest -bin_PROGRAMS += zcash-gtest +noinst_PROGRAMS += zcash-gtest # tool for generating our public parameters. # test_checktransaction.cpp MUST be before diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 191b2d8a7..09d34043a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -1,5 +1,5 @@ TESTS += test/test_bitcoin -bin_PROGRAMS += test/test_bitcoin +noinst_PROGRAMS += test/test_bitcoin TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) diff --git a/src/Makefile.zcash.include b/src/Makefile.zcash.include index 59aeb861f..562512dfe 100644 --- a/src/Makefile.zcash.include +++ b/src/Makefile.zcash.include @@ -1,4 +1,4 @@ -bin_PROGRAMS += \ +noinst_PROGRAMS += \ zcash/GenerateParams \ zcash/CreateJoinSplit From fce7260811cea4cb3fc3d423a559664b42a27909 Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 3 Jan 2017 12:50:19 -0800 Subject: [PATCH 20/56] Fixes #1964 to catch general exception in z_sendmany and catch exceptions as reference-to-const. --- src/wallet/asyncrpcoperation_sendmany.cpp | 13 ++++++++----- src/wallet/rpcwallet.cpp | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/wallet/asyncrpcoperation_sendmany.cpp b/src/wallet/asyncrpcoperation_sendmany.cpp index c65664eae..4cd7ab804 100644 --- a/src/wallet/asyncrpcoperation_sendmany.cpp +++ b/src/wallet/asyncrpcoperation_sendmany.cpp @@ -86,7 +86,7 @@ AsyncRPCOperation_sendmany::AsyncRPCOperation_sendmany( isfromzaddr_ = true; frompaymentaddress_ = addr; spendingkey_ = key; - } catch (std::runtime_error e) { + } catch (const std::runtime_error& e) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, string("runtime error: ") + e.what()); } } @@ -106,17 +106,20 @@ void AsyncRPCOperation_sendmany::main() { try { success = main_impl(); - } catch (Object objError) { + } catch (const Object& objError) { int code = find_value(objError, "code").get_int(); std::string message = find_value(objError, "message").get_str(); set_error_code(code); set_error_message(message); - } catch (runtime_error e) { + } catch (const runtime_error& e) { set_error_code(-1); set_error_message("runtime error: " + string(e.what())); - } catch (logic_error e) { + } catch (const logic_error& e) { set_error_code(-1); set_error_message("logic error: " + string(e.what())); + } catch (const exception& e) { + set_error_code(-1); + set_error_message("general exception: " + string(e.what())); } catch (...) { set_error_code(-2); set_error_message("unknown error"); @@ -577,7 +580,7 @@ bool AsyncRPCOperation_sendmany::main_impl() { FormatMoney(plaintext.value, false) ); - } catch (const std::exception e) { + } catch (const std::exception& e) { throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Error decrypting output note of previous JoinSplit: %s", e.what())); } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..dc1bd9aff 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2955,7 +2955,7 @@ Value z_listreceivedbyaddress(const Array& params, bool fHelp) CZCPaymentAddress address(fromaddress); try { zaddr = address.Get(); - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid zaddr."); } @@ -3022,7 +3022,7 @@ Value z_getbalance(const Array& params, bool fHelp) CZCPaymentAddress address(fromaddress); try { zaddr = address.Get(); - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid from address, should be a taddr or zaddr."); } if (!pwalletMain->HaveSpendingKey(zaddr)) { @@ -3226,7 +3226,7 @@ Value z_sendmany(const Array& params, bool fHelp) CZCPaymentAddress address(fromaddress); try { zaddr = address.Get(); - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { // invalid throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid from address, should be a taddr or zaddr."); } @@ -3273,7 +3273,7 @@ Value z_sendmany(const Array& params, bool fHelp) CZCPaymentAddress zaddr(address); zaddr.Get(); isZaddr = true; - } catch (std::runtime_error) { + } catch (const std::runtime_error&) { throw JSONRPCError(RPC_INVALID_PARAMETER, string("Invalid parameter, unknown address format: ")+address ); } } From 01f6c5dbee57913314a89914e9c714b14fc9546c Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 3 Jan 2017 23:57:11 -0800 Subject: [PATCH 21/56] Fixes #1967 by adding age of note to z_sendmany logging. --- src/wallet/asyncrpcoperation_sendmany.cpp | 29 +++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/wallet/asyncrpcoperation_sendmany.cpp b/src/wallet/asyncrpcoperation_sendmany.cpp index c65664eae..a4d32361a 100644 --- a/src/wallet/asyncrpcoperation_sendmany.cpp +++ b/src/wallet/asyncrpcoperation_sendmany.cpp @@ -448,13 +448,22 @@ bool AsyncRPCOperation_sendmany::main_impl() { info.notes.push_back(note); outPoints.push_back(outPoint); - - LogPrint("zrpc", "%s: spending note (txid=%s, vjoinsplit=%d, ciphertext=%d, amount=%s)\n", + int wtxHeight = -1; + int wtxDepth = -1; + { + LOCK2(cs_main, pwalletMain->cs_wallet); + const CWalletTx& wtx = pwalletMain->mapWallet[outPoint.hash]; + wtxHeight = mapBlockIndex[wtx.hashBlock]->nHeight; + wtxDepth = wtx.GetDepthInMainChain(); + } + LogPrint("zrpc", "%s: spending note (txid=%s, vjoinsplit=%d, ciphertext=%d, amount=%s, height=%d, confirmations=%d)\n", getId().substr(0, 10), outPoint.hash.ToString().substr(0, 10), outPoint.js, int(outPoint.n), // uint8_t - FormatMoney(noteFunds, false) + FormatMoney(noteFunds, false), + wtxHeight, + wtxDepth ); @@ -611,12 +620,22 @@ bool AsyncRPCOperation_sendmany::main_impl() { jsInputValue += noteFunds; - LogPrint("zrpc", "%s: spending note (txid=%s, vjoinsplit=%d, ciphertext=%d, amount=%s)\n", + int wtxHeight = -1; + int wtxDepth = -1; + { + LOCK2(cs_main, pwalletMain->cs_wallet); + const CWalletTx& wtx = pwalletMain->mapWallet[jso.hash]; + wtxHeight = mapBlockIndex[wtx.hashBlock]->nHeight; + wtxDepth = wtx.GetDepthInMainChain(); + } + LogPrint("zrpc", "%s: spending note (txid=%s, vjoinsplit=%d, ciphertext=%d, amount=%s, height=%d, confirmations=%d)\n", getId().substr(0, 10), jso.hash.ToString().substr(0, 10), jso.js, int(jso.n), // uint8_t - FormatMoney(noteFunds, false) + FormatMoney(noteFunds, false), + wtxHeight, + wtxDepth ); } From 7e3a20f235f83650e7792c0da7bedb7bc92ff0fb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 4 Jan 2017 15:40:35 +0100 Subject: [PATCH 22/56] Scan the whole chain whenever a z-key is imported Closes #1941. --- src/wallet/rpcdump.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 789bd4537..d5f9a3785 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -575,6 +575,9 @@ Value z_importkey(const Array& params, bool fHelp) pwalletMain->mapZKeyMetadata[addr].nCreateTime = 1; + // whenever a key is imported, we need to scan the whole chain + pwalletMain->nTimeFirstKey = 1; // 0 would be considered 'no value' + // We want to scan for transactions and notes if (fRescan) { pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true); From bddf4079fd07762e3588c6c5928648c1d6c422f5 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 4 Jan 2017 17:58:07 +0100 Subject: [PATCH 23/56] Instruct users to run zcash-fetch-params if network params aren't available Closes #1786. --- src/init.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 53f6fb832..6b87a5870 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -621,6 +621,17 @@ static void ZC_LoadParams() boost::filesystem::path pk_path = ZC_GetParamsDir() / "sprout-proving.key"; boost::filesystem::path vk_path = ZC_GetParamsDir() / "sprout-verifying.key"; + if (!(boost::filesystem::exists(pk_path) && boost::filesystem::exists(vk_path))) { + uiInterface.ThreadSafeMessageBox(strprintf( + _("Cannot find the Zcash network parameters in the following directory:\n" + "%s\n" + "Please run 'zcash-fetch-params' and then restart."), + ZC_GetParamsDir()), + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); + return; + } + pzcashParams = ZCJoinSplit::Unopened(); LogPrintf("Loading verifying key from %s\n", vk_path.string().c_str()); From 4c1cd287beafb0b22c74fd229247a559722dd0f3 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 4 Jan 2017 12:17:23 -0800 Subject: [PATCH 24/56] Fixes a bug where the unsigned transaction was logged by z_sendmany after a successful sign and send, meaning that the logged hash fragment would be different from the txid logged by "AddToWallet". This issue occured when sending from transparent addresses, as utxo inputs must be signed. It did not occur when sending from shielded addresses. --- src/wallet/asyncrpcoperation_sendmany.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/wallet/asyncrpcoperation_sendmany.cpp b/src/wallet/asyncrpcoperation_sendmany.cpp index c65664eae..6fa30d07d 100644 --- a/src/wallet/asyncrpcoperation_sendmany.cpp +++ b/src/wallet/asyncrpcoperation_sendmany.cpp @@ -776,6 +776,12 @@ void AsyncRPCOperation_sendmany::sign_send_raw_transaction(Object obj) o.push_back(Pair("hex", signedtxn)); set_result(Value(o)); } + + // Keep the signed transaction so we can hash to the same txid + CDataStream stream(ParseHex(signedtxn), SER_NETWORK, PROTOCOL_VERSION); + CTransaction tx; + stream >> tx; + tx_ = tx; } From 83561c9cb314a677172c8038a25a3265f4991ba8 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Jan 2017 10:21:00 +0100 Subject: [PATCH 25/56] Trigger metrics UI refresh on new messages --- src/metrics.cpp | 15 +++++++++++++-- src/metrics.h | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index 053538660..2803a3130 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -20,6 +20,7 @@ CCriticalSection cs_metrics; boost::synchronized_value nNodeStartTime; +boost::synchronized_value nNextRefresh; AtomicCounter transactionsValidated; AtomicCounter ehSolverRuns; AtomicCounter solutionTargetChecks; @@ -60,6 +61,13 @@ double GetLocalSolPS() return GetLocalSolPS_INTERNAL(GetUptime()); } +void TriggerRefresh() +{ + *nNextRefresh = GetTime(); + // Ensure that the refresh has started before we return + MilliSleep(200); +} + static bool metrics_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) @@ -85,6 +93,9 @@ static bool metrics_ThreadSafeMessageBox(const std::string& message, if (u->size() > 5) { u->pop_back(); } + + TriggerRefresh(); + return false; } static void metrics_InitMessage(const std::string& message) @@ -333,8 +344,8 @@ void ThreadShowMetricsScreen() std::cout << "----------------------------------------" << std::endl; } - int64_t nWaitEnd = GetTime() + nRefresh; - while (GetTime() < nWaitEnd) { + *nNextRefresh = GetTime() + nRefresh; + while (GetTime() < *nNextRefresh) { boost::this_thread::interruption_point(); MilliSleep(200); } diff --git a/src/metrics.h b/src/metrics.h index 2851178b6..57264a7b5 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -34,6 +34,8 @@ void TrackMinedBlock(uint256 hash); void MarkStartTime(); double GetLocalSolPS(); +void TriggerRefresh(); + void ConnectMetricsScreen(); void ThreadShowMetricsScreen(); From e698459ec882a77dab8257e17a95f6e2630e5217 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Jan 2017 10:26:49 +0100 Subject: [PATCH 26/56] Strip out the SECURE flag in metrics UI so message style is detected --- src/metrics.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/metrics.cpp b/src/metrics.cpp index 2803a3130..df0016113 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -72,6 +72,9 @@ static bool metrics_ThreadSafeMessageBox(const std::string& message, const std::string& caption, unsigned int style) { + // The SECURE flag has no effect in the metrics UI. + style &= ~CClientUIInterface::SECURE; + std::string strCaption; // Check for usage of predefined caption switch (style) { From fc31ea8739bb9496f649bc30b2ce4ee76279c0cd Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Thu, 5 Jan 2017 00:31:22 -0700 Subject: [PATCH 27/56] Add 'CreateJoinSplit' standalone utility to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 17922996d..713859560 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ src/qt/test/test_bitcoin-qt # Zcash utilities src/zcash/GenerateParams +src/zcash/CreateJoinSplit *zcashTest.pk *zcashTest.vk From b6e439b2610b7d78d58ed566473f8c291bd0384c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 5 Jan 2017 08:38:35 +0100 Subject: [PATCH 28/56] Handle newlines in UI messages --- src/metrics.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index df0016113..9f3795e00 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -261,8 +261,21 @@ int printMessageBox(size_t cols) std::cout << _("Messages:") << std::endl; for (auto it = u->cbegin(); it != u->cend(); ++it) { std::cout << *it << std::endl; - // Handle wrapped lines - lines += (it->size() / cols); + // Handle newlines and wrapped lines + size_t i = 0; + size_t j = 0; + while (j < it->size()) { + i = it->find('\n', j); + if (i == std::string::npos) { + i = it->size(); + } else { + // Newline + lines++; + } + // Wrapped lines + lines += ((i-j) / cols); + j = i + 1; + } } std::cout << std::endl; return lines; From 0b0a48c88149180f93c2f869f684ae1feffde73c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 5 Jan 2017 11:38:38 +0100 Subject: [PATCH 29/56] Suggest ./zcutil/fetch-params.sh as well Once we improve the from-source installation docs to use 'make install', we can revert this commit. --- src/init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.cpp b/src/init.cpp index 6b87a5870..f85899667 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -625,7 +625,7 @@ static void ZC_LoadParams() uiInterface.ThreadSafeMessageBox(strprintf( _("Cannot find the Zcash network parameters in the following directory:\n" "%s\n" - "Please run 'zcash-fetch-params' and then restart."), + "Please run 'zcash-fetch-params' or './zcutil/fetch-params.sh' and then restart."), ZC_GetParamsDir()), "", CClientUIInterface::MSG_ERROR); StartShutdown(); From 53e9a6a0e2966b4a51cf9a5a5e4df11eea3c7210 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 5 Jan 2017 11:56:42 +0100 Subject: [PATCH 30/56] Update debug categories Closes #1954. --- src/init.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 53f6fb832..a048b9def 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -376,11 +376,12 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0)); } - string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below + string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, lock, mempool, net, partitioncheck, pow, proxy, prune, " + "rand, reindex, rpc, selectcoins, zrpc, zrpcunsafe"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=", strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + - _("If is not supplied or if = 1, output all debugging information.") + _(" can be:") + " " + debugCategories + "."); + _("If is not supplied or if = 1, output all debugging information.") + " " + _(" can be:") + " " + debugCategories + "."); #ifdef ENABLE_WALLET strUsage += HelpMessageOpt("-gen", strprintf(_("Generate coins (default: %u)"), 0)); strUsage += HelpMessageOpt("-genproclimit=", strprintf(_("Set the number of threads for coin generation if enabled (-1 = all cores, default: %d)"), 1)); From 894a0b32952793f3a82610dae20f8613c8d79230 Mon Sep 17 00:00:00 2001 From: Eran Tromer Date: Thu, 5 Jan 2017 11:25:41 -0500 Subject: [PATCH 31/56] CreateJoinSplit: add start_profiling() call This solves the problem of profiling output displaying nonsensical large time values. --- src/zcash/CreateJoinSplit.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/zcash/CreateJoinSplit.cpp b/src/zcash/CreateJoinSplit.cpp index 65dc7f65c..ea2d59c9e 100644 --- a/src/zcash/CreateJoinSplit.cpp +++ b/src/zcash/CreateJoinSplit.cpp @@ -5,11 +5,14 @@ #include "../util.h" #include "primitives/transaction.h" #include "zcash/JoinSplit.hpp" +#include "libsnark/common/profiling.hpp" using namespace libzcash; int main(int argc, char **argv) { + libsnark::start_profiling(); + auto p = ZCJoinSplit::Unopened(); p->loadVerifyingKey((ZC_GetParamsDir() / "sprout-verifying.key").string()); p->setProvingKeyPath((ZC_GetParamsDir() / "sprout-proving.key").string()); From e957192c545750b2a21910b59387ab45c831e8ae Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 7 Jul 2015 14:53:48 +0200 Subject: [PATCH 32/56] rpc: Implement random-cookie based authentication When no `-rpcpassword` is specified, use a special 'cookie' file for authentication. This file is generated with random content when the daemon starts, and deleted when it exits. Read access to this file controls who can access through RPC. By default this file is stored in the data directory but it be overriden with `-rpccookiefile`. This is similar to Tor CookieAuthentication: see https://www.torproject.org/docs/tor-manual.html.en Alternative to #6258. Like that pull, this allows running bitcoind without any manual configuration. However, daemons should ideally never write to their configuration files, so I prefer this solution. --- src/bitcoin-cli.cpp | 24 ++++++++++------ src/rpcprotocol.cpp | 67 +++++++++++++++++++++++++++++++++++++++++++++ src/rpcprotocol.h | 10 +++++++ src/rpcserver.cpp | 34 +++++++++-------------- 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 4954a44a2..ab92fb747 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -96,12 +96,6 @@ static bool AppInitRPC(int argc, char* argv[]) Object CallRPC(const string& strMethod, const Array& params) { - if (mapArgs["-rpcuser"] == "" && mapArgs["-rpcpassword"] == "") - throw runtime_error(strprintf( - _("You must set rpcpassword= in the configuration file:\n%s\n" - "If the file does not exist, create it with owner-readable-only file permissions."), - GetConfigFile().string().c_str())); - // Connect to localhost bool fUseSSL = GetBoolArg("-rpcssl", false); boost::asio::io_service io_service; @@ -115,10 +109,24 @@ Object CallRPC(const string& strMethod, const Array& params) if (!fConnected) throw CConnectionFailed("couldn't connect to server"); + // Find credentials to use + std::string strRPCUserColonPass; + if (mapArgs["-rpcpassword"] == "") { + // Try fall back to cookie-based authentication if no password is provided + if (!GetAuthCookie(&strRPCUserColonPass)) { + throw runtime_error(strprintf( + _("You must set rpcpassword= in the configuration file:\n%s\n" + "If the file does not exist, create it with owner-readable-only file permissions."), + GetConfigFile().string().c_str())); + + } + } else { + strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; + } + // HTTP basic authentication - string strUserPass64 = EncodeBase64(mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]); map mapRequestHeaders; - mapRequestHeaders["Authorization"] = string("Basic ") + strUserPass64; + mapRequestHeaders["Authorization"] = string("Basic ") + EncodeBase64(strRPCUserColonPass); // Send request string strRequest = JSONRPCRequest(strMethod, params, 1); diff --git a/src/rpcprotocol.cpp b/src/rpcprotocol.cpp index b60b3e4fd..19badb473 100644 --- a/src/rpcprotocol.cpp +++ b/src/rpcprotocol.cpp @@ -6,6 +6,7 @@ #include "rpcprotocol.h" #include "clientversion.h" +#include "random.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" @@ -13,6 +14,7 @@ #include "version.h" #include +#include #include #include @@ -288,3 +290,68 @@ Object JSONRPCError(int code, const string& message) error.push_back(Pair("message", message)); return error; } + +/** Username used when cookie authentication is in use (arbitrary, only for + * recognizability in debugging/logging purposes) + */ +static const std::string COOKIEAUTH_USER = "__cookie__"; +/** Default name for auth cookie file */ +static const std::string COOKIEAUTH_FILE = ".cookie"; + +boost::filesystem::path GetAuthCookieFile() +{ + boost::filesystem::path path(GetArg("-rpccookiefile", COOKIEAUTH_FILE)); + if (!path.is_complete()) path = GetDataDir() / path; + return path; +} + +bool GenerateAuthCookie(std::string *cookie_out) +{ + unsigned char rand_pwd[32]; + GetRandBytes(rand_pwd, 32); + std::string cookie = COOKIEAUTH_USER + ":" + EncodeBase64(&rand_pwd[0],32); + + /** the umask determines what permissions are used to create this file - + * these are set to 077 in init.cpp unless overridden with -sysperms. + */ + std::ofstream file; + boost::filesystem::path filepath = GetAuthCookieFile(); + file.open(filepath.string().c_str()); + if (!file.is_open()) { + LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath.string()); + return false; + } + file << cookie; + file.close(); + LogPrintf("Generated RPC authentication cookie %s\n", filepath.string()); + + if (cookie_out) + *cookie_out = cookie; + return true; +} + +bool GetAuthCookie(std::string *cookie_out) +{ + std::ifstream file; + std::string cookie; + boost::filesystem::path filepath = GetAuthCookieFile(); + file.open(filepath.string().c_str()); + if (!file.is_open()) + return false; + std::getline(file, cookie); + file.close(); + + if (cookie_out) + *cookie_out = cookie; + return true; +} + +void DeleteAuthCookie() +{ + try { + boost::filesystem::remove(GetAuthCookieFile()); + } catch (const boost::filesystem::filesystem_error& e) { + LogPrintf("%s: Unable to remove random auth cookie file: %s\n", __func__, e.what()); + } +} + diff --git a/src/rpcprotocol.h b/src/rpcprotocol.h index aac3502a6..a58a77fe0 100644 --- a/src/rpcprotocol.h +++ b/src/rpcprotocol.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "json/json_spirit_reader_template.h" #include "json/json_spirit_utils.h" @@ -165,4 +166,13 @@ json_spirit::Object JSONRPCReplyObj(const json_spirit::Value& result, const json std::string JSONRPCReply(const json_spirit::Value& result, const json_spirit::Value& error, const json_spirit::Value& id); json_spirit::Object JSONRPCError(int code, const std::string& message); +/** Get name of RPC authentication cookie file */ +boost::filesystem::path GetAuthCookieFile(); +/** Generate a new RPC authentication cookie and write it to disk */ +bool GenerateAuthCookie(std::string *cookie_out); +/** Read the RPC authentication cookie from disk */ +bool GetAuthCookie(std::string *cookie_out); +/** Delete RPC authentication cookie from disk */ +void DeleteAuthCookie(); + #endif // BITCOIN_RPCPROTOCOL_H diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index e354e91a4..72d4eec97 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -619,28 +619,18 @@ void StartRPCThreads() strAllowed += subnet.ToString() + " "; LogPrint("rpc", "Allowing RPC connections from: %s\n", strAllowed); - strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; - if (((mapArgs["-rpcpassword"] == "") || - (mapArgs["-rpcuser"] == mapArgs["-rpcpassword"])) && Params().RequireRPCPassword()) + if (mapArgs["-rpcpassword"] == "") { - unsigned char rand_pwd[32]; - GetRandBytes(rand_pwd, 32); - uiInterface.ThreadSafeMessageBox(strprintf( - _("To use zcashd you must set an rpcpassword in the configuration file:\n" - "%s\n" - "It is recommended you use the following random password:\n" - "rpcuser=zcashrpc\n" - "rpcpassword=%s\n" - "(you do not need to remember this password)\n" - "The username and password MUST NOT be the same.\n" - "If the file does not exist, create it with owner-readable-only file permissions.\n" - "It is also recommended to set alertnotify so you are notified of problems;\n" - "for example: alertnotify=echo %%s | mail -s \"Zcash Alert\" admin@foo.com\n"), - GetConfigFile().string(), - EncodeBase58(&rand_pwd[0],&rand_pwd[0]+32)), - "", CClientUIInterface::MSG_ERROR | CClientUIInterface::SECURE); - StartShutdown(); - return; + LogPrintf("No rpcpassword set - using random cookie authentication\n"); + if (!GenerateAuthCookie(&strRPCUserColonPass)) { + uiInterface.ThreadSafeMessageBox( + _("Error: A fatal internal error occured, see debug.log for details"), // Same message as AbortNode + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); + return; + } + } else { + strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; } assert(rpc_io_service == NULL); @@ -806,6 +796,8 @@ void StopRPCThreads() } deadlineTimers.clear(); + DeleteAuthCookie(); + rpc_io_service->stop(); g_rpcSignals.Stopped(); if (rpc_worker_group != NULL) From 1c76501909ce0fcef8b8ac1e7b36cb5b7cd9fe08 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 9 Jan 2017 18:54:08 +0100 Subject: [PATCH 33/56] Rename build-aux/m4/bitcoin_find_bdb48.m4 to remove version Closes #1622. --- build-aux/m4/{bitcoin_find_bdb48.m4 => bitcoin_find_bdb.m4} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build-aux/m4/{bitcoin_find_bdb48.m4 => bitcoin_find_bdb.m4} (100%) diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb.m4 similarity index 100% rename from build-aux/m4/bitcoin_find_bdb48.m4 rename to build-aux/m4/bitcoin_find_bdb.m4 From e466df9369ba3c0f4776d4294dc751cf02bffb27 Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 9 Jan 2017 23:23:36 -0800 Subject: [PATCH 34/56] Bump COPYRIGHT_YEAR from 2016 to 2017. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a6e40ab5a..bdcd3e00d 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ define(_CLIENT_VERSION_BUILD, 50) define(_ZC_BUILD_VAL, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, m4_incr(_CLIENT_VERSION_BUILD), m4_eval(_CLIENT_VERSION_BUILD < 50), 1, m4_eval(_CLIENT_VERSION_BUILD - 24), m4_eval(_CLIENT_VERSION_BUILD == 50), 1, , m4_eval(_CLIENT_VERSION_BUILD - 50))) define(_CLIENT_VERSION_SUFFIX, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, _CLIENT_VERSION_REVISION-beta$1, m4_eval(_CLIENT_VERSION_BUILD < 50), 1, _CLIENT_VERSION_REVISION-rc$1, m4_eval(_CLIENT_VERSION_BUILD == 50), 1, _CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION-$1))) define(_CLIENT_VERSION_IS_RELEASE, true) -define(_COPYRIGHT_YEAR, 2016) +define(_COPYRIGHT_YEAR, 2017) AC_INIT([Zcash],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_SUFFIX(_ZC_BUILD_VAL)],[https://github.com/zcash/zcash/issues],[zcash]) AC_CONFIG_SRCDIR([src/main.cpp]) AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) From d87f00c4d5e9f547481ccb3977febdc6f740828e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 10 Jan 2017 16:33:41 +0100 Subject: [PATCH 35/56] Throw an error if zcash.conf is missing An empty zcash.conf is sufficient to bypass this error. --- src/util.cpp | 2 +- src/util.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/util.cpp b/src/util.cpp index b667acc5e..8fbdfc65b 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -534,7 +534,7 @@ void ReadConfigFile(map& mapSettingsRet, { boost::filesystem::ifstream streamConfig(GetConfigFile()); if (!streamConfig.good()) - return; // No zcash.conf file is OK + throw missing_zcash_conf(); set setOptions; setOptions.insert("*"); diff --git a/src/util.h b/src/util.h index ef3a347fa..b7d255e4d 100644 --- a/src/util.h +++ b/src/util.h @@ -127,6 +127,10 @@ boost::filesystem::path GetConfigFile(); boost::filesystem::path GetPidFile(); void CreatePidFile(const boost::filesystem::path &path, pid_t pid); #endif +class missing_zcash_conf : public std::runtime_error { +public: + missing_zcash_conf() : std::runtime_error("Missing zcash.conf") { } +}; void ReadConfigFile(std::map& mapSettingsRet, std::map >& mapMultiSettingsRet); #ifdef WIN32 boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate = true); From 24f4e3365bb3aed558fe0baa80d17715aad5b3db Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 11 Jan 2017 12:55:35 +0100 Subject: [PATCH 36/56] Show a friendly message explaining why zcashd needs a zcash.conf --- src/bitcoind.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 2ec14ccc6..8634be211 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -97,6 +97,24 @@ bool AppInit(int argc, char* argv[]) try { ReadConfigFile(mapArgs, mapMultiArgs); + } catch (const missing_zcash_conf& e) { + fprintf(stderr, + (_("Before starting zcashd, you need to create a configuration file:\n" + "%s\n" + "It can be completely empty! That indicates you are happy with the default\n" + "configuration of zcashd. But requiring a configuration file to start ensures\n" + "that zcashd won't accidentally compromise your privacy if there was a default\n" + "option you needed to change.\n" + "\n" + "You can look at the example configuration file for suggestions of default\n" + "options that you may want to change. It should be in one of these locations,\n" + "depending on how you installed Zcash:\n") + + _("- Source code: %s\n" + "- .deb package: %s\n")).c_str(), + GetConfigFile().string().c_str(), + "contrib/DEBIAN/examples/zcash.conf", + "/usr/share/doc/zcash/examples/zcash.conf"); + return false; } catch (const std::exception& e) { fprintf(stderr,"Error reading configuration file: %s\n", e.what()); return false; From 2d2f3d180655b806d331c6a1b7066f6a35393347 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 11 Jan 2017 11:49:10 -0800 Subject: [PATCH 37/56] Closes #1780. Result of z_getoperationstatus now sorted by creation time of operation --- src/wallet/rpcwallet.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..39c540a0c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3172,6 +3172,13 @@ Value z_getoperationstatus_IMPL(const Array& params, bool fRemoveFinishedOperati } } + // sort results chronologically by creation_time + std::sort(ret.begin(), ret.end(), [](Value a, Value b) -> bool { + const int64_t t1 = find_value(a.get_obj(), "creation_time").get_int64(); + const int64_t t2 = find_value(b.get_obj(), "creation_time").get_int64(); + return t1 < t2; + }); + return ret; } From 56a033219ccb2d2adc0f5617439be4005f757130 Mon Sep 17 00:00:00 2001 From: Paige Peterson Date: Wed, 11 Jan 2017 16:55:29 -0500 Subject: [PATCH 38/56] Create ISSUE_TEMPLATE.md --- ISSUE_TEMPLATE.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..3efa2d3ad --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,41 @@ + + +This issue tracker is only for technical issues related to zcashd. + +General zcash questions and/or support requests and are best directed to the [Zcash Forums](https://forum.z.cash) or [Community Rocket.Chat](https://chat.zcashcommunity.com). + +For reporting security vulnerabilities or for sensitive discussions with our security team, please contact [security@z.cash](mailto:security@z.cash). You can use the [GPG key](https://z.cash/gpg-pubkeys/security.asc) (fingerprint: `AF85 0445 546C 18B7 86F9 2C62 88FB 8B86 D8B5 A68C`) to send an encrypted message. The key and fingerprint are duplicated on our [Public Keys page](https://z.cash/support/pubkeys.html). + +### Describe the issue + +### Can you reliably reproduce the issue? +#### If so, please list the steps to reproduce below: +1. +2. +3. + +### Expected behaviour +Tell us what should happen + +### Actual behaviour + errors +Tell us what happens instead including any noticable error output (any messages displayed on-screen when e.g. a crash occurred) + +### The version of Zcash you were using: +Run `zcashd --version` to find out + +### Machine specs: +- OS name + version: +- CPU: +- RAM: +- Disk size: +- Disk Type (HD/SDD): +- Linux kernel version (uname -a): +- Compiler version (gcc -version): + +### Any extra information that might be useful in the debugging process. +This includes the relevant contents of `~/.zcash/debug.log`. Raw text or a link to a pastebin type site are preferred. +Please also include any non-standard things you did during compilation (extra flags, dependency version changes etc.) if applicable. + +### Do you have a back up of `~/.zcash directory` and/or take a VM snapshot? +- Backing up / making a copy of the `~/.zcash` directory might help make the problem reproducible. Please redact appropriately. +- Taking a VM snapshot is really helpful for interactively testing fixes From 80f3db55d813e73aa916e27e7b34c5c25707064e Mon Sep 17 00:00:00 2001 From: Paige Peterson Date: Thu, 12 Jan 2017 16:42:19 -0500 Subject: [PATCH 39/56] move template to subdirectory, fix typo, include prompt under describing issue section, include uploading file directly to github ticket as option for sharing logs --- ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename ISSUE_TEMPLATE.md => .github/ISSUE_TEMPLATE.md (82%) diff --git a/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md similarity index 82% rename from ISSUE_TEMPLATE.md rename to .github/ISSUE_TEMPLATE.md index 3efa2d3ad..7cec0d649 100644 --- a/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -2,11 +2,12 @@ This issue tracker is only for technical issues related to zcashd. -General zcash questions and/or support requests and are best directed to the [Zcash Forums](https://forum.z.cash) or [Community Rocket.Chat](https://chat.zcashcommunity.com). +General Zcash questions and/or support requests and are best directed to the [Zcash Forums](https://forum.z.cash) or [Community Rocket.Chat](https://chat.zcashcommunity.com). For reporting security vulnerabilities or for sensitive discussions with our security team, please contact [security@z.cash](mailto:security@z.cash). You can use the [GPG key](https://z.cash/gpg-pubkeys/security.asc) (fingerprint: `AF85 0445 546C 18B7 86F9 2C62 88FB 8B86 D8B5 A68C`) to send an encrypted message. The key and fingerprint are duplicated on our [Public Keys page](https://z.cash/support/pubkeys.html). ### Describe the issue +Please provide a general summary of the issue you're experiencing ### Can you reliably reproduce the issue? #### If so, please list the steps to reproduce below: @@ -33,9 +34,9 @@ Run `zcashd --version` to find out - Compiler version (gcc -version): ### Any extra information that might be useful in the debugging process. -This includes the relevant contents of `~/.zcash/debug.log`. Raw text or a link to a pastebin type site are preferred. +This includes the relevant contents of `~/.zcash/debug.log`. You can paste raw text, attach the file directly in the issue or link to the text via a pastebin type site. Please also include any non-standard things you did during compilation (extra flags, dependency version changes etc.) if applicable. -### Do you have a back up of `~/.zcash directory` and/or take a VM snapshot? +### Do you have a back up of `~/.zcash` directory and/or take a VM snapshot? - Backing up / making a copy of the `~/.zcash` directory might help make the problem reproducible. Please redact appropriately. - Taking a VM snapshot is really helpful for interactively testing fixes From f1498d58298496991cc70ced86b5396eac846488 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 12 Jan 2017 18:26:20 -0800 Subject: [PATCH 40/56] Remove UTF-8 BOM efbbbf from zcash.conf to avoid problems with command line tools --- contrib/DEBIAN/examples/zcash.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/DEBIAN/examples/zcash.conf b/contrib/DEBIAN/examples/zcash.conf index e9935a872..cbfdf4ee7 100644 --- a/contrib/DEBIAN/examples/zcash.conf +++ b/contrib/DEBIAN/examples/zcash.conf @@ -1,4 +1,4 @@ -## +## ## zcash.conf configuration file. Lines beginning with # are comments. ## From af02114469d1b5734c3f8a3096562217459ac4da Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 9 Jan 2017 23:17:48 -0800 Subject: [PATCH 41/56] Closes #1097 so zcash-cli now displays license info like zcashd. LicenseInfo is refactored from init.cpp to util.cpp so that the bitcoin-cli makefile target does not need to be modified. --- src/bitcoin-cli.cpp | 2 ++ src/init.cpp | 13 ------------- src/init.h | 2 -- src/util.cpp | 13 +++++++++++++ src/util.h | 3 +++ 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 4954a44a2..16cda9dc0 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -71,6 +71,8 @@ static bool AppInitRPC(int argc, char* argv[]) " zcash-cli [options] help " + _("Get help for a command") + "\n"; strUsage += "\n" + HelpMessageCli(); + } else { + strUsage += LicenseInfo(); } fprintf(stdout, "%s", strUsage.c_str()); diff --git a/src/init.cpp b/src/init.cpp index 53f6fb832..9a7fd8829 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -460,19 +460,6 @@ std::string HelpMessage(HelpMessageMode mode) return strUsage; } -std::string LicenseInfo() -{ - return FormatParagraph(strprintf(_("Copyright (C) 2009-%i The Bitcoin Core Developers"), COPYRIGHT_YEAR)) + "\n" + - FormatParagraph(strprintf(_("Copyright (C) 2015-%i The Zcash Developers"), COPYRIGHT_YEAR)) + "\n" + - "\n" + - FormatParagraph(_("This is experimental software.")) + "\n" + - "\n" + - FormatParagraph(_("Distributed under the MIT software license, see the accompanying file COPYING or .")) + "\n" + - "\n" + - FormatParagraph(_("This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit and cryptographic software written by Eric Young and UPnP software written by Thomas Bernard.")) + - "\n"; -} - static void BlockNotifyCallback(const uint256& hashNewTip) { std::string strCmd = GetArg("-blocknotify", ""); diff --git a/src/init.h b/src/init.h index cfc88255b..34290b276 100644 --- a/src/init.h +++ b/src/init.h @@ -34,7 +34,5 @@ enum HelpMessageMode { /** Help for options shared between UI and daemon (for -help) */ std::string HelpMessage(HelpMessageMode mode); -/** Returns licensing information (for -version) */ -std::string LicenseInfo(); #endif // BITCOIN_INIT_H diff --git a/src/util.cpp b/src/util.cpp index b667acc5e..b38f67ada 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -815,3 +815,16 @@ void SetThreadPriority(int nPriority) #endif // PRIO_THREAD #endif // WIN32 } + +std::string LicenseInfo() +{ + return FormatParagraph(strprintf(_("Copyright (C) 2009-%i The Bitcoin Core Developers"), COPYRIGHT_YEAR)) + "\n" + + FormatParagraph(strprintf(_("Copyright (C) 2015-%i The Zcash Developers"), COPYRIGHT_YEAR)) + "\n" + + "\n" + + FormatParagraph(_("This is experimental software.")) + "\n" + + "\n" + + FormatParagraph(_("Distributed under the MIT software license, see the accompanying file COPYING or .")) + "\n" + + "\n" + + FormatParagraph(_("This product includes software developed by the OpenSSL Project for use in the OpenSSL Toolkit and cryptographic software written by Eric Young and UPnP software written by Thomas Bernard.")) + + "\n"; +} diff --git a/src/util.h b/src/util.h index ef3a347fa..bbfafaf60 100644 --- a/src/util.h +++ b/src/util.h @@ -135,6 +135,9 @@ boost::filesystem::path GetTempPath(); void ShrinkDebugFile(); void runCommand(std::string strCommand); +/** Returns licensing information (for -version) */ +std::string LicenseInfo(); + inline bool IsSwitchChar(char c) { #ifdef WIN32 From 9064d73bf8b22f6b9449f72fc6559f25ca45b1fc Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 9 Jan 2017 21:25:42 -0800 Subject: [PATCH 42/56] Fixes #1497 ZCA-009 by restricting data exporting to user defined folder. Previously the RPC interface allowed z_exportwallet, backupwallet and dumpwallet to write data to an arbitrary filename. ZCA-009 demonstrates how this is vulnerable. The resolution is to only allow data to written when the -exportdir has been configured. Also filenames are restricted to alphanumeric characters. --- src/init.cpp | 1 + src/test/rpc_wallet_tests.cpp | 22 ++++++++++++++++------ src/util.cpp | 20 ++++++++++++++++++++ src/util.h | 1 + src/utilstrencodings.cpp | 16 ++++++++++++++++ src/utilstrencodings.h | 1 + src/wallet/rpcdump.cpp | 29 +++++++++++++++++++++++++---- src/wallet/rpcwallet.cpp | 31 ++++++++++++++++++++++++------- 8 files changed, 104 insertions(+), 17 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 53f6fb832..8a2398be1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -289,6 +289,7 @@ std::string HelpMessage(HelpMessageMode mode) #endif } strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); + strUsage += HelpMessageOpt("-exportdir=", _("Specify directory to be used when exporting data")); strUsage += HelpMessageOpt("-dbcache=", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file") + " " + _("on startup")); strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); diff --git a/src/test/rpc_wallet_tests.cpp b/src/test/rpc_wallet_tests.cpp index 8471ebebc..d0ee27aea 100644 --- a/src/test/rpc_wallet_tests.cpp +++ b/src/test/rpc_wallet_tests.cpp @@ -358,16 +358,26 @@ BOOST_AUTO_TEST_CASE(rpc_wallet_z_exportwallet) pwalletMain->GetPaymentAddresses(addrs); BOOST_CHECK(addrs.size()==1); + // Set up paths + boost::filesystem::path tmppath = boost::filesystem::temp_directory_path(); + boost::filesystem::path tmpfilename = boost::filesystem::unique_path("%%%%%%%%"); + boost::filesystem::path exportfilepath = tmppath / tmpfilename; + + // export will fail since exportdir is not set + BOOST_CHECK_THROW(CallRPC(string("z_exportwallet ") + tmpfilename.string()), runtime_error); + + // set exportdir + mapArgs["-exportdir"] = tmppath.native(); + + // run some tests BOOST_CHECK_THROW(CallRPC("z_exportwallet"), runtime_error); BOOST_CHECK_THROW(CallRPC("z_exportwallet toomany args"), runtime_error); - - boost::filesystem::path temp = boost::filesystem::temp_directory_path() / - boost::filesystem::unique_path(); - const std::string path = temp.native(); + BOOST_CHECK_THROW(CallRPC(string("z_exportwallet invalid!*/_chars.txt")), runtime_error); + + BOOST_CHECK_NO_THROW(CallRPC(string("z_exportwallet ") + tmpfilename.string())); - BOOST_CHECK_NO_THROW(CallRPC(string("z_exportwallet ") + path)); auto addr = paymentAddress.Get(); libzcash::SpendingKey key; @@ -382,7 +392,7 @@ BOOST_AUTO_TEST_CASE(rpc_wallet_z_exportwallet) EnsureWalletIsUnlocked(); ifstream file; - file.open(path.c_str(), std::ios::in | std::ios::ate); + file.open(exportfilepath.string().c_str(), std::ios::in | std::ios::ate); BOOST_CHECK(file.is_open()); bool fVerified = false; int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); diff --git a/src/util.cpp b/src/util.cpp index b667acc5e..ec8f90445 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -484,6 +484,26 @@ const boost::filesystem::path &ZC_GetParamsDir() return path; } +// Return the user specified export directory. Create directory if it doesn't exist. +// If user did not set option, return an empty path. +// If there is a filesystem problem, throw an exception. +const boost::filesystem::path GetExportDir() +{ + namespace fs = boost::filesystem; + fs::path path; + if (mapArgs.count("-exportdir")) { + path = fs::system_complete(mapArgs["-exportdir"]); + if (fs::exists(path) && !fs::is_directory(path)) { + throw std::runtime_error(strprintf("The -exportdir '%s' already exists and is not a directory", path.string())); + } + if (!fs::exists(path) && !fs::create_directories(path)) { + throw std::runtime_error(strprintf("Failed to create directory at -exportdir '%s'", path.string())); + } + } + return path; +} + + const boost::filesystem::path &GetDataDir(bool fNetSpecific) { namespace fs = boost::filesystem; diff --git a/src/util.h b/src/util.h index ef3a347fa..56c7ea39b 100644 --- a/src/util.h +++ b/src/util.h @@ -134,6 +134,7 @@ boost::filesystem::path GetSpecialFolderPath(int nFolder, bool fCreate = true); boost::filesystem::path GetTempPath(); void ShrinkDebugFile(); void runCommand(std::string strCommand); +const boost::filesystem::path GetExportDir(); inline bool IsSwitchChar(char c) { diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index c15bddc6f..646536f1e 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -30,6 +30,22 @@ string SanitizeString(const string& str) return strResult; } +string SanitizeFilename(const string& str) +{ + /** + * safeChars chosen to restrict filename, keeping it simple to avoid cross-platform issues. + * http://stackoverflow.com/a/2306003 + */ + static string safeChars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890"); + string strResult; + for (std::string::size_type i = 0; i < str.size(); i++) + { + if (safeChars.find(str[i]) != std::string::npos) + strResult.push_back(str[i]); + } + return strResult; +} + const signed char p_util_hexdigit[256] = { -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index b0edd8b54..e08de362b 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -22,6 +22,7 @@ /** This is needed because the foreach macro can't get over the comma in pair */ #define PAIRTYPE(t1, t2) std::pair +std::string SanitizeFilename(const std::string& str); std::string SanitizeString(const std::string& str); std::vector ParseHex(const char* psz); std::vector ParseHex(const std::string& str); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 789bd4537..07d81bce5 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -430,7 +430,9 @@ Value z_exportwallet(const Array& params, bool fHelp) "z_exportwallet \"filename\"\n" "\nExports all wallet keys, for taddr and zaddr, in a human-readable format.\n" "\nArguments:\n" - "1. \"filename\" (string, required) The filename\n" + "1. \"filename\" (string, required) The filename, saved in folder set by zcashd -exportdir option\n" + "\nResult:\n" + "\"path\" (string) The full path of the destination file\n" "\nExamples:\n" + HelpExampleCli("z_exportwallet", "\"test\"") + HelpExampleRpc("z_exportwallet", "\"test\"") @@ -449,7 +451,9 @@ Value dumpwallet(const Array& params, bool fHelp) "dumpwallet \"filename\"\n" "\nDumps taddr wallet keys in a human-readable format.\n" "\nArguments:\n" - "1. \"filename\" (string, required) The filename\n" + "1. \"filename\" (string, required) The filename, saved in folder set by zcashd -exportdir option\n" + "\nResult:\n" + "\"path\" (string) The full path of the destination file\n" "\nExamples:\n" + HelpExampleCli("dumpwallet", "\"test\"") + HelpExampleRpc("dumpwallet", "\"test\"") @@ -464,8 +468,24 @@ Value dumpwallet_impl(const Array& params, bool fHelp, bool fDumpZKeys) EnsureWalletIsUnlocked(); + boost::filesystem::path exportdir; + try { + exportdir = GetExportDir(); + } catch (const std::runtime_error& e) { + throw JSONRPCError(RPC_INTERNAL_ERROR, e.what()); + } + if (exportdir.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot export wallet until the zcashd -exportdir option has been set"); + } + std::string unclean = params[0].get_str(); + std::string clean = SanitizeFilename(unclean); + if (clean.compare(unclean) != 0) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Filename is invalid as only alphanumeric characters are allowed. Try '%s' instead.", clean)); + } + boost::filesystem::path exportfilepath = exportdir / clean; + ofstream file; - file.open(params[0].get_str().c_str()); + file.open(exportfilepath.string().c_str()); if (!file.is_open()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); @@ -523,7 +543,8 @@ Value dumpwallet_impl(const Array& params, bool fHelp, bool fDumpZKeys) file << "# End of dump\n"; file.close(); - return Value::null; + + return exportfilepath.string(); } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..770f5826c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1766,21 +1766,38 @@ Value backupwallet(const Array& params, bool fHelp) if (fHelp || params.size() != 1) throw runtime_error( "backupwallet \"destination\"\n" - "\nSafely copies wallet.dat to destination, which can be a directory or a path with filename.\n" + "\nSafely copies wallet.dat to destination filename\n" "\nArguments:\n" - "1. \"destination\" (string) The destination directory or file\n" + "1. \"destination\" (string, required) The destination filename, saved in the directory set by -exportdir option.\n" + "\nResult:\n" + "\"path\" (string) The full path of the destination file\n" "\nExamples:\n" - + HelpExampleCli("backupwallet", "\"backup.dat\"") - + HelpExampleRpc("backupwallet", "\"backup.dat\"") + + HelpExampleCli("backupwallet", "\"backupdata\"") + + HelpExampleRpc("backupwallet", "\"backupdata\"") ); LOCK2(cs_main, pwalletMain->cs_wallet); - string strDest = params[0].get_str(); - if (!BackupWallet(*pwalletMain, strDest)) + boost::filesystem::path exportdir; + try { + exportdir = GetExportDir(); + } catch (const std::runtime_error& e) { + throw JSONRPCError(RPC_INTERNAL_ERROR, e.what()); + } + if (exportdir.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot backup wallet until the -exportdir option has been set"); + } + std::string unclean = params[0].get_str(); + std::string clean = SanitizeFilename(unclean); + if (clean.compare(unclean) != 0) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Filename is invalid as only alphanumeric characters are allowed. Try '%s' instead.", clean)); + } + boost::filesystem::path exportfilepath = exportdir / clean; + + if (!BackupWallet(*pwalletMain, exportfilepath.string())) throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); - return Value::null; + return exportfilepath.string(); } From f32cade8fd5065c519aacb1f681c8aad8c2dc7cf Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 9 Jan 2017 10:00:14 -0800 Subject: [PATCH 43/56] Closes #1957 by adding tx serialization size to listtransactions output. --- src/wallet/rpcwallet.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..91c2dee41 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1325,6 +1325,7 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe entry.push_back(Pair("fee", ValueFromAmount(-nFee))); if (fLong) WalletTxToJSON(wtx, entry); + entry.push_back(Pair("size", static_cast(wtx).GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION))); ret.push_back(entry); } } @@ -1361,6 +1362,7 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe entry.push_back(Pair("vout", r.vout)); if (fLong) WalletTxToJSON(wtx, entry); + entry.push_back(Pair("size", static_cast(wtx).GetSerializeSize(SER_NETWORK, PROTOCOL_VERSION))); ret.push_back(entry); } } @@ -1429,6 +1431,7 @@ Value listtransactions(const Array& params, bool fHelp) " \"otheraccount\": \"accountname\", (string) For the 'move' category of transactions, the account the funds came \n" " from (for receiving funds, positive amounts), or went to (for sending funds,\n" " negative amounts).\n" + " \"size\": n, (numeric) Transaction size in bytes\n" " }\n" "]\n" From 5ae347827f0c9074a26f01bb6fe73083a0a8b689 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 16 Jan 2017 20:16:29 +0100 Subject: [PATCH 44/56] Fix gtest ordering broken by #1949 Part of #1539 --- src/Makefile.gtest.include | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 1c9203ffa..1ebb0c0f9 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -10,7 +10,14 @@ zcash_gtest_SOURCES = \ gtest/test_checktransaction.cpp \ gtest/json_test_vectors.cpp \ gtest/json_test_vectors.h \ - gtest/test_foundersreward.cpp \ + gtest/test_foundersreward.cpp +# These tests are order-dependent, because they +# depend on global state (see #1539) +if ENABLE_WALLET +zcash_gtest_SOURCES += \ + wallet/gtest/test_wallet_zkeys.cpp +endif +zcash_gtest_SOURCES += \ gtest/test_jsonspirit.cpp \ gtest/test_tautology.cpp \ gtest/test_equihash.cpp \ @@ -30,8 +37,7 @@ zcash_gtest_SOURCES = \ gtest/test_checkblock.cpp if ENABLE_WALLET zcash_gtest_SOURCES += \ - wallet/gtest/test_wallet.cpp \ - wallet/gtest/test_wallet_zkeys.cpp + wallet/gtest/test_wallet.cpp endif zcash_gtest_CPPFLAGS = -DMULTICORE -fopenmp -DBINARY_OUTPUT -DCURVE_ALT_BN128 -DSTATIC From 8aa7937d44d5f2cf300a3d30c53d57a51a0b624f Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 7 Jan 2017 07:41:42 -0800 Subject: [PATCH 45/56] Fixes #1960: z_getoperationstatus/result now includes operation details. --- doc/payment-api.md | 4 ++++ qa/rpc-tests/wallet_protectcoinbase.py | 8 ++++++++ src/asyncrpcoperation.h | 3 ++- src/wallet/asyncrpcoperation_sendmany.cpp | 19 +++++++++++++++++-- src/wallet/asyncrpcoperation_sendmany.h | 6 +++++- src/wallet/rpcwallet.cpp | 10 +++++++++- 6 files changed, 45 insertions(+), 5 deletions(-) diff --git a/doc/payment-api.md b/doc/payment-api.md index 3560add64..69877e5db 100644 --- a/doc/payment-api.md +++ b/doc/payment-api.md @@ -89,6 +89,10 @@ Asynchronous calls return an OperationStatus object which is a JSON object with * code : number * message: error message +Depending on the type of asynchronous call, there may be other key-value pairs. For example, a z_sendmany operation will also include the following in an OperationStatus object: + +* params : an object containing the parameters to z_sendmany + Currently, as soon as you retrieve the operation status for an operation which has finished, that is it has either succeeded, failed, or been cancelled, the operation and any associated information is removed. It is currently not possible to cancel operations. diff --git a/qa/rpc-tests/wallet_protectcoinbase.py b/qa/rpc-tests/wallet_protectcoinbase.py index 5f0c285c0..ddef5fdaf 100755 --- a/qa/rpc-tests/wallet_protectcoinbase.py +++ b/qa/rpc-tests/wallet_protectcoinbase.py @@ -98,6 +98,14 @@ class WalletProtectCoinbaseTest (BitcoinTestFramework): else: status = results[0]["status"] errorString = results[0]["error"]["message"] + + # Test that the returned status object contains a params field with the operation's input parameters + params =results[0]["params"] + assert_equal(params["fee"], Decimal('0.0001')) # default + assert_equal(params["minconf"], Decimal('1')) # default + assert_equal(params["fromaddress"], mytaddr) + assert_equal(params["amounts"][0]["address"], myzaddr) + assert_equal(params["amounts"][0]["amount"], Decimal('1.23456789')) break assert_equal("failed", status) assert_equal("wallet does not allow any change" in errorString, True) diff --git a/src/asyncrpcoperation.h b/src/asyncrpcoperation.h index bcfda892e..526a62ad0 100644 --- a/src/asyncrpcoperation.h +++ b/src/asyncrpcoperation.h @@ -66,7 +66,8 @@ public: return creation_time_; } - Value getStatus() const; + // Override this method to add data to the default status object. + virtual Value getStatus() const; Value getError() const; diff --git a/src/wallet/asyncrpcoperation_sendmany.cpp b/src/wallet/asyncrpcoperation_sendmany.cpp index c65664eae..c9b5b3ef2 100644 --- a/src/wallet/asyncrpcoperation_sendmany.cpp +++ b/src/wallet/asyncrpcoperation_sendmany.cpp @@ -51,8 +51,9 @@ AsyncRPCOperation_sendmany::AsyncRPCOperation_sendmany( std::vector tOutputs, std::vector zOutputs, int minDepth, - CAmount fee) : - fromaddress_(fromAddress), t_outputs_(tOutputs), z_outputs_(zOutputs), mindepth_(minDepth), fee_(fee) + CAmount fee, + Value contextInfo) : + fromaddress_(fromAddress), t_outputs_(tOutputs), z_outputs_(zOutputs), mindepth_(minDepth), fee_(fee), contextinfo_(contextInfo) { assert(fee_ > 0); @@ -1089,4 +1090,18 @@ boost::array AsyncRPCOperation_sendmany::get_memo_f return memo; } +/** + * Override getStatus() to append the operation's input parameters to the default status object. + */ +Value AsyncRPCOperation_sendmany::getStatus() const { + Value v = AsyncRPCOperation::getStatus(); + if (contextinfo_.is_null()) { + return v; + } + + Object obj = v.get_obj(); + obj.push_back(Pair("method", "z_sendmany")); + obj.push_back(Pair("params", contextinfo_ )); + return Value(obj); +} diff --git a/src/wallet/asyncrpcoperation_sendmany.h b/src/wallet/asyncrpcoperation_sendmany.h index 0382f0241..5108de8c5 100644 --- a/src/wallet/asyncrpcoperation_sendmany.h +++ b/src/wallet/asyncrpcoperation_sendmany.h @@ -50,7 +50,7 @@ struct WitnessAnchorData { class AsyncRPCOperation_sendmany : public AsyncRPCOperation { public: - AsyncRPCOperation_sendmany(std::string fromAddress, std::vector tOutputs, std::vector zOutputs, int minDepth, CAmount fee = ASYNC_RPC_OPERATION_DEFAULT_MINERS_FEE); + AsyncRPCOperation_sendmany(std::string fromAddress, std::vector tOutputs, std::vector zOutputs, int minDepth, CAmount fee = ASYNC_RPC_OPERATION_DEFAULT_MINERS_FEE, Value contextInfo = Value::null); virtual ~AsyncRPCOperation_sendmany(); // We don't want to be copied or moved around @@ -61,11 +61,15 @@ public: virtual void main(); + virtual Value getStatus() const; + bool testmode = false; // Set to true to disable sending txs and generating proofs private: friend class TEST_FRIEND_AsyncRPCOperation_sendmany; // class for unit testing + Value contextinfo_; // optional data to include in return value from getStatus() + CAmount fee_; int mindepth_; std::string fromaddress_; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6b0f5f528..143c28f65 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3353,9 +3353,17 @@ Value z_sendmany(const Array& params, bool fHelp) } } + // Use input parameters as the optional context info to be returned by z_getoperationstatus and z_getoperationresult. + Object o; + o.push_back(Pair("fromaddress", params[0])); + o.push_back(Pair("amounts", params[1])); + o.push_back(Pair("minconf", nMinDepth)); + o.push_back(Pair("fee", std::stod(FormatMoney(nFee)))); + Value contextInfo = Value(o); + // Create operation and add to global queue std::shared_ptr q = getAsyncRPCQueue(); - std::shared_ptr operation( new AsyncRPCOperation_sendmany(fromaddress, taddrRecipients, zaddrRecipients, nMinDepth, nFee) ); + std::shared_ptr operation( new AsyncRPCOperation_sendmany(fromaddress, taddrRecipients, zaddrRecipients, nMinDepth, nFee, contextInfo) ); q->addOperation(operation); AsyncRPCOperationId operationId = operation->getId(); return operationId; From 6a62bff3e0e7aede256893bf1743b4698d85610c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Jan 2017 13:11:55 +0100 Subject: [PATCH 46/56] Debian package lint - Tweak description synopsis to make Debian happy - Put bash completion files in correct directory - Add a manpage for zcash-fetch-params --- contrib/DEBIAN/control | 2 +- contrib/DEBIAN/manpages/zcash-fetch-params.1 | 28 ++++++++++++++++++++ zcutil/build-debian-package.sh | 4 ++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 contrib/DEBIAN/manpages/zcash-fetch-params.1 diff --git a/contrib/DEBIAN/control b/contrib/DEBIAN/control index b24ffc54d..02eb73328 100644 --- a/contrib/DEBIAN/control +++ b/contrib/DEBIAN/control @@ -13,7 +13,7 @@ Package: zcash Version: 1.0.4 Architecture: amd64 Depends: libgomp1 -Description: An implementation of the "Zerocash" protocol. +Description: HTTPS for money. Based on Bitcoin's code, it intends to offer a far higher standard of privacy and anonymity through a sophisticiated zero-knowledge proving scheme which preserves confidentiality of transaction metadata. diff --git a/contrib/DEBIAN/manpages/zcash-fetch-params.1 b/contrib/DEBIAN/manpages/zcash-fetch-params.1 new file mode 100644 index 000000000..76dab4c23 --- /dev/null +++ b/contrib/DEBIAN/manpages/zcash-fetch-params.1 @@ -0,0 +1,28 @@ +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. +.TH ZCASH-FETCH-PARAMS "1" "January 2017" "Zcash - zcash-fetch-params" "User Commands" +.SH NAME +zcash-fetch-params \- Downloads the Zcash network parameters +.SH DESCRIPTION +Zcash \- zcash-fetch\-params +.PP +This script will fetch the Zcash zkSNARK parameters and verify their +integrity with sha256sum. +.PP +If they already exist locally, it will exit now and do nothing else. +.PP +This script will fetch the Zcash zkSNARK parameters and verify their +integrity with sha256sum. +.PP +If they already exist locally, it will exit now and do nothing else. +.SH "SEE ALSO" +The full documentation for +.B Zcash +is maintained as a Texinfo manual. If the +.B info +and +.B Zcash +programs are properly installed at your site, the command +.IP +.B info Zcash +.PP +should give you access to the complete manual. diff --git a/zcutil/build-debian-package.sh b/zcutil/build-debian-package.sh index 8d063399e..39fbfe67b 100755 --- a/zcutil/build-debian-package.sh +++ b/zcutil/build-debian-package.sh @@ -23,8 +23,8 @@ if [ -d $BUILD_DIR ]; then rm -R $BUILD_DIR fi -DEB_CMP=$BUILD_DIR/etc/bash_completion.d DEB_BIN=$BUILD_DIR/usr/bin +DEB_CMP=$BUILD_DIR/usr/share/bash-completion/completions DEB_DOC=$BUILD_DIR/usr/share/doc/$PACKAGE_NAME DEB_MAN=$BUILD_DIR/usr/share/man/man1 mkdir -p $BUILD_DIR/DEBIAN $DEB_CMP $DEB_BIN $DEB_DOC $DEB_MAN @@ -49,6 +49,7 @@ cp -r $SRC_DEB/examples $DEB_DOC # Copy manpages cp $SRC_DEB/manpages/zcashd.1 $DEB_MAN cp $SRC_DEB/manpages/zcash-cli.1 $DEB_MAN +cp $SRC_DEB/manpages/zcash-fetch-params.1 $DEB_MAN # Copy bash completion files cp $SRC_PATH/contrib/bitcoind.bash-completion $DEB_CMP/zcashd cp $SRC_PATH/contrib/bitcoin-cli.bash-completion $DEB_CMP/zcash-cli @@ -57,6 +58,7 @@ gzip --best -n $DEB_DOC/changelog gzip --best -n $DEB_DOC/changelog.Debian gzip --best -n $DEB_MAN/zcashd.1 gzip --best -n $DEB_MAN/zcash-cli.1 +gzip --best -n $DEB_MAN/zcash-fetch-params.1 # Create the Debian package fakeroot dpkg-deb --build $BUILD_DIR From b50b4d5c38db9914817ff11f8fa623b64273c8e9 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Jan 2017 16:17:21 +0100 Subject: [PATCH 47/56] Generate Debian control file to fix shlibs lint --- .gitignore | 3 +++ contrib/{DEBIAN => debian}/changelog | 0 contrib/{DEBIAN => debian}/compat | 0 contrib/{DEBIAN => debian}/control | 10 +++++----- contrib/{DEBIAN => debian}/copyright | 0 contrib/{DEBIAN => debian}/examples/zcash.conf | 0 contrib/{DEBIAN => debian}/manpages/zcash-cli.1 | 0 .../{DEBIAN => debian}/manpages/zcash-fetch-params.1 | 0 contrib/{DEBIAN => debian}/manpages/zcashd.1 | 0 contrib/{DEBIAN => debian}/postinst | 0 contrib/{DEBIAN => debian}/postrm | 0 contrib/{DEBIAN => debian}/preinst | 0 contrib/{DEBIAN => debian}/prerm | 0 contrib/{DEBIAN => debian}/rules | 0 contrib/{DEBIAN => debian}/zcash.examples | 0 contrib/{DEBIAN => debian}/zcash.install | 0 contrib/{DEBIAN => debian}/zcash.manpages | 0 doc/release-process.md | 1 - zcutil/build-debian-package.sh | 12 ++++++++---- 19 files changed, 16 insertions(+), 10 deletions(-) rename contrib/{DEBIAN => debian}/changelog (100%) rename contrib/{DEBIAN => debian}/compat (100%) rename contrib/{DEBIAN => debian}/control (85%) rename contrib/{DEBIAN => debian}/copyright (100%) rename contrib/{DEBIAN => debian}/examples/zcash.conf (100%) rename contrib/{DEBIAN => debian}/manpages/zcash-cli.1 (100%) rename contrib/{DEBIAN => debian}/manpages/zcash-fetch-params.1 (100%) rename contrib/{DEBIAN => debian}/manpages/zcashd.1 (100%) rename contrib/{DEBIAN => debian}/postinst (100%) rename contrib/{DEBIAN => debian}/postrm (100%) rename contrib/{DEBIAN => debian}/preinst (100%) rename contrib/{DEBIAN => debian}/prerm (100%) rename contrib/{DEBIAN => debian}/rules (100%) rename contrib/{DEBIAN => debian}/zcash.examples (100%) rename contrib/{DEBIAN => debian}/zcash.install (100%) rename contrib/{DEBIAN => debian}/zcash.manpages (100%) diff --git a/.gitignore b/.gitignore index 713859560..34d16b302 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,6 @@ qa/pull-tester/test.*/* /doc/doxygen/ libzcashconsensus.pc + +contrib/debian/files +contrib/debian/substvars diff --git a/contrib/DEBIAN/changelog b/contrib/debian/changelog similarity index 100% rename from contrib/DEBIAN/changelog rename to contrib/debian/changelog diff --git a/contrib/DEBIAN/compat b/contrib/debian/compat similarity index 100% rename from contrib/DEBIAN/compat rename to contrib/debian/compat diff --git a/contrib/DEBIAN/control b/contrib/debian/control similarity index 85% rename from contrib/DEBIAN/control rename to contrib/debian/control index 02eb73328..b0c220cf0 100644 --- a/contrib/DEBIAN/control +++ b/contrib/debian/control @@ -3,16 +3,16 @@ Section: utils Priority: optional Maintainer: Zcash Company Homepage: https://z.cash -Build-Depends: autoconf, automake, bsdmainutils, build-essential - git, g++-multilib, libc6-dev, libtool - m4, ncurses-dev, pkg-config, python +Build-Depends: autoconf, automake, bsdmainutils, build-essential, + git, g++-multilib, libc6-dev, libtool, + m4, ncurses-dev, pkg-config, python, unzip, wget, zlib1g-dev Vcs-Git: https://github.com/zcash/zcash.git Vcs-Browser: https://github.com/zcash/zcash + Package: zcash -Version: 1.0.4 Architecture: amd64 -Depends: libgomp1 +Depends: ${shlibs:Depends} Description: HTTPS for money. Based on Bitcoin's code, it intends to offer a far higher standard of privacy and anonymity through a sophisticiated zero-knowledge diff --git a/contrib/DEBIAN/copyright b/contrib/debian/copyright similarity index 100% rename from contrib/DEBIAN/copyright rename to contrib/debian/copyright diff --git a/contrib/DEBIAN/examples/zcash.conf b/contrib/debian/examples/zcash.conf similarity index 100% rename from contrib/DEBIAN/examples/zcash.conf rename to contrib/debian/examples/zcash.conf diff --git a/contrib/DEBIAN/manpages/zcash-cli.1 b/contrib/debian/manpages/zcash-cli.1 similarity index 100% rename from contrib/DEBIAN/manpages/zcash-cli.1 rename to contrib/debian/manpages/zcash-cli.1 diff --git a/contrib/DEBIAN/manpages/zcash-fetch-params.1 b/contrib/debian/manpages/zcash-fetch-params.1 similarity index 100% rename from contrib/DEBIAN/manpages/zcash-fetch-params.1 rename to contrib/debian/manpages/zcash-fetch-params.1 diff --git a/contrib/DEBIAN/manpages/zcashd.1 b/contrib/debian/manpages/zcashd.1 similarity index 100% rename from contrib/DEBIAN/manpages/zcashd.1 rename to contrib/debian/manpages/zcashd.1 diff --git a/contrib/DEBIAN/postinst b/contrib/debian/postinst similarity index 100% rename from contrib/DEBIAN/postinst rename to contrib/debian/postinst diff --git a/contrib/DEBIAN/postrm b/contrib/debian/postrm similarity index 100% rename from contrib/DEBIAN/postrm rename to contrib/debian/postrm diff --git a/contrib/DEBIAN/preinst b/contrib/debian/preinst similarity index 100% rename from contrib/DEBIAN/preinst rename to contrib/debian/preinst diff --git a/contrib/DEBIAN/prerm b/contrib/debian/prerm similarity index 100% rename from contrib/DEBIAN/prerm rename to contrib/debian/prerm diff --git a/contrib/DEBIAN/rules b/contrib/debian/rules similarity index 100% rename from contrib/DEBIAN/rules rename to contrib/debian/rules diff --git a/contrib/DEBIAN/zcash.examples b/contrib/debian/zcash.examples similarity index 100% rename from contrib/DEBIAN/zcash.examples rename to contrib/debian/zcash.examples diff --git a/contrib/DEBIAN/zcash.install b/contrib/debian/zcash.install similarity index 100% rename from contrib/DEBIAN/zcash.install rename to contrib/debian/zcash.install diff --git a/contrib/DEBIAN/zcash.manpages b/contrib/debian/zcash.manpages similarity index 100% rename from contrib/DEBIAN/zcash.manpages rename to contrib/debian/zcash.manpages diff --git a/doc/release-process.md b/doc/release-process.md index 93b97adaf..cfbf4fad1 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -35,7 +35,6 @@ previous release: README.md src/clientversion.h configure.ac - contrib/DEBIAN/control contrib/gitian-descriptors/gitian-linux.yml Build and commit to update versions, and then perform the following commands: diff --git a/zcutil/build-debian-package.sh b/zcutil/build-debian-package.sh index 39fbfe67b..55e07e689 100755 --- a/zcutil/build-debian-package.sh +++ b/zcutil/build-debian-package.sh @@ -8,7 +8,7 @@ set -x BUILD_PATH="/tmp/zcbuild" PACKAGE_NAME="zcash" SRC_PATH=`pwd` -SRC_DEB=$SRC_PATH/contrib/DEBIAN +SRC_DEB=$SRC_PATH/contrib/debian umask 022 @@ -16,7 +16,7 @@ if [ ! -d $BUILD_PATH ]; then mkdir $BUILD_PATH fi -PACKAGE_VERSION=$(grep Version $SRC_PATH/contrib/DEBIAN/control | cut -d: -f2 | tr -d ' ') +PACKAGE_VERSION=$($SRC_PATH/src/zcashd --version | grep version | cut -d' ' -f4 | tr -d v) BUILD_DIR="$BUILD_PATH/$PACKAGE_NAME-$PACKAGE_VERSION-amd64" if [ -d $BUILD_DIR ]; then @@ -30,8 +30,6 @@ DEB_MAN=$BUILD_DIR/usr/share/man/man1 mkdir -p $BUILD_DIR/DEBIAN $DEB_CMP $DEB_BIN $DEB_DOC $DEB_MAN chmod 0755 -R $BUILD_DIR/* -# Copy control file -cp $SRC_DEB/control $BUILD_DIR/DEBIAN # Package maintainer scripts (currently empty) #cp $SRC_DEB/postinst $BUILD_DIR/DEBIAN #cp $SRC_DEB/postrm $BUILD_DIR/DEBIAN @@ -60,6 +58,12 @@ gzip --best -n $DEB_MAN/zcashd.1 gzip --best -n $DEB_MAN/zcash-cli.1 gzip --best -n $DEB_MAN/zcash-fetch-params.1 +cd $SRC_PATH/contrib + +# Create the control file +dpkg-shlibdeps $DEB_BIN/zcashd $DEB_BIN/zcash-cli +dpkg-gencontrol -P$BUILD_DIR + # Create the Debian package fakeroot dpkg-deb --build $BUILD_DIR cp $BUILD_PATH/$PACKAGE_NAME-$PACKAGE_VERSION-amd64.deb $SRC_PATH From 336fa630207321e387bf3dd865dbd24985dfcd77 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 17 Jan 2017 13:19:49 +0100 Subject: [PATCH 48/56] Create empty zcash.conf during performance measurements --- qa/zcash/performance-measurements.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/zcash/performance-measurements.sh b/qa/zcash/performance-measurements.sh index 851adef09..8f29e397f 100755 --- a/qa/zcash/performance-measurements.sh +++ b/qa/zcash/performance-measurements.sh @@ -5,7 +5,7 @@ set -e DATADIR=./benchmark-datadir function zcash_rpc { - ./src/zcash-cli -rpcwait -rpcuser=user -rpcpassword=password -rpcport=5983 "$@" + ./src/zcash-cli -datadir="$DATADIR" -rpcwait -rpcuser=user -rpcpassword=password -rpcport=5983 "$@" } function zcashd_generate { @@ -15,6 +15,7 @@ function zcashd_generate { function zcashd_start { rm -rf "$DATADIR" mkdir -p "$DATADIR" + touch "$DATADIR/zcash.conf" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & ZCASHD_PID=$! } @@ -27,6 +28,7 @@ function zcashd_stop { function zcashd_massif_start { rm -rf "$DATADIR" mkdir -p "$DATADIR" + touch "$DATADIR/zcash.conf" rm -f massif.out valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & ZCASHD_PID=$! @@ -41,6 +43,7 @@ function zcashd_massif_stop { function zcashd_valgrind_start { rm -rf "$DATADIR" mkdir -p "$DATADIR" + touch "$DATADIR/zcash.conf" rm -f valgrind.out valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & ZCASHD_PID=$! From 9847bc870dce362e9fb703d2f221756e334febd7 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 18 Jan 2017 12:11:50 +0100 Subject: [PATCH 49/56] Create empty zcash.conf during coverage checks Fixes regression caused by #2013. --- qa/pull-tester/run-bitcoind-for-test.sh.in | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/pull-tester/run-bitcoind-for-test.sh.in b/qa/pull-tester/run-bitcoind-for-test.sh.in index b0b8811a0..999b0f5b7 100755 --- a/qa/pull-tester/run-bitcoind-for-test.sh.in +++ b/qa/pull-tester/run-bitcoind-for-test.sh.in @@ -7,6 +7,7 @@ ZCASH_LOAD_TIMEOUT=500 DATADIR="@abs_top_builddir@/.zcash" rm -rf "$DATADIR" mkdir -p "$DATADIR"/regtest +touch "$DATADIR/zcash.conf" touch "$DATADIR/regtest/debug.log" tail -q -n 1 -F "$DATADIR/regtest/debug.log" | grep -m 1 -q "Done loading" & WAITER=$! From 9773b95f6e51c5964fa12f33427bc35980a7b4ab Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 18 Jan 2017 13:57:05 +0100 Subject: [PATCH 50/56] Coverage build system tweaks --- Makefile.am | 10 ++++++++++ src/Makefile.am | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 2b584b039..9e146d836 100644 --- a/Makefile.am +++ b/Makefile.am @@ -158,6 +158,8 @@ baseline_filtered.info: baseline.info "$(abs_builddir)/depends/x86_64-unknown-linux-gnu/include/gtest/*" \ "$(abs_builddir)/src/gtest/*" \ "$(abs_builddir)/src/test/*" \ + "$(abs_builddir)/src/wallet/gtest/*" \ + "$(abs_builddir)/src/wallet/test/*" \ -o $@ leveldb_baseline.info: baseline_filtered.info @@ -171,6 +173,8 @@ leveldb_baseline_filtered.info: leveldb_baseline.info "$(abs_builddir)/depends/x86_64-unknown-linux-gnu/include/gtest/*" \ "$(abs_builddir)/src/gtest/*" \ "$(abs_builddir)/src/test/*" \ + "$(abs_builddir)/src/wallet/gtest/*" \ + "$(abs_builddir)/src/wallet/test/*" \ -o $@ baseline_filtered_combined.info: leveldb_baseline_filtered.info baseline_filtered.info @@ -190,6 +194,8 @@ test_bitcoin_filtered.info: test_bitcoin.info "$(abs_builddir)/depends/x86_64-unknown-linux-gnu/include/gtest/*" \ "$(abs_builddir)/src/gtest/*" \ "$(abs_builddir)/src/test/*" \ + "$(abs_builddir)/src/wallet/gtest/*" \ + "$(abs_builddir)/src/wallet/test/*" \ -o $@ zcash-gtest.info: baseline_filtered_combined.info @@ -206,6 +212,8 @@ zcash-gtest_filtered.info: zcash-gtest.info "$(abs_builddir)/depends/x86_64-unknown-linux-gnu/include/gtest/*" \ "$(abs_builddir)/src/gtest/*" \ "$(abs_builddir)/src/test/*" \ + "$(abs_builddir)/src/wallet/gtest/*" \ + "$(abs_builddir)/src/wallet/test/*" \ -o $@ block_test.info: test_bitcoin_filtered.info @@ -223,6 +231,8 @@ block_test_filtered.info: block_test.info "$(abs_builddir)/depends/x86_64-unknown-linux-gnu/include/gtest/*" \ "$(abs_builddir)/src/gtest/*" \ "$(abs_builddir)/src/test/*" \ + "$(abs_builddir)/src/wallet/gtest/*" \ + "$(abs_builddir)/src/wallet/test/*" \ -o $@ test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info diff --git a/src/Makefile.am b/src/Makefile.am index 78684a4eb..f18071678 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -483,7 +483,7 @@ libzcashconsensus_la_CPPFLAGS = $(CRYPTO_CFLAGS) -I$(builddir)/obj -DBUILD_BITCO endif # -CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a *.gcda *.gcno +CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a *.gcda *.gcno */*.gcno wallet/*/*.gcno DISTCLEANFILES = obj/build.h From bab89e35c737e62f309c5c1d93f1a446f6af56dd Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 18 Jan 2017 10:05:49 -0800 Subject: [PATCH 51/56] Update walletbackup.py qa test to use -exportdir option --- qa/rpc-tests/walletbackup.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/qa/rpc-tests/walletbackup.py b/qa/rpc-tests/walletbackup.py index e71065781..5f3eb6fcc 100755 --- a/qa/rpc-tests/walletbackup.py +++ b/qa/rpc-tests/walletbackup.py @@ -47,8 +47,13 @@ class WalletBackupTest(BitcoinTestFramework): # This mirrors how the network was setup in the bash test def setup_network(self, split=False): + # -exportdir option means we must provide a valid path to the destination folder for wallet backups + ed0 = "-exportdir=" + self.options.tmpdir + "/node0" + ed1 = "-exportdir=" + self.options.tmpdir + "/node1" + ed2 = "-exportdir=" + self.options.tmpdir + "/node2" + # nodes 1, 2,3 are spenders, let's give them a keypool=100 - extra_args = [["-keypool=100"], ["-keypool=100"], ["-keypool=100"], []] + extra_args = [["-keypool=100", ed0], ["-keypool=100", ed1], ["-keypool=100", ed2], []] self.nodes = start_nodes(4, self.options.tmpdir, extra_args) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) @@ -122,12 +127,12 @@ class WalletBackupTest(BitcoinTestFramework): logging.info("Backing up") tmpdir = self.options.tmpdir - self.nodes[0].backupwallet(tmpdir + "/node0/wallet.bak") - self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.dump") - self.nodes[1].backupwallet(tmpdir + "/node1/wallet.bak") - self.nodes[1].dumpwallet(tmpdir + "/node1/wallet.dump") - self.nodes[2].backupwallet(tmpdir + "/node2/wallet.bak") - self.nodes[2].dumpwallet(tmpdir + "/node2/wallet.dump") + self.nodes[0].backupwallet("walletbak") + self.nodes[0].dumpwallet("walletdump") + self.nodes[1].backupwallet("walletbak") + self.nodes[1].dumpwallet("walletdump") + self.nodes[2].backupwallet("walletbak") + self.nodes[2].dumpwallet("walletdump") logging.info("More transactions") for i in range(5): @@ -159,9 +164,9 @@ class WalletBackupTest(BitcoinTestFramework): shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") # Restore wallets from backup - shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node0/walletbak", tmpdir + "/node0/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node1/walletbak", tmpdir + "/node1/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node2/walletbak", tmpdir + "/node2/regtest/wallet.dat") logging.info("Re-starting nodes") self.start_three() @@ -185,9 +190,9 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) - self.nodes[0].importwallet(tmpdir + "/node0/wallet.dump") - self.nodes[1].importwallet(tmpdir + "/node1/wallet.dump") - self.nodes[2].importwallet(tmpdir + "/node2/wallet.dump") + self.nodes[0].importwallet(tmpdir + "/node0/walletdump") + self.nodes[1].importwallet(tmpdir + "/node1/walletdump") + self.nodes[2].importwallet(tmpdir + "/node2/walletdump") sync_blocks(self.nodes) From bcbde86a87d01f54d111042c5752ee9ad81df0e4 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 18 Jan 2017 10:52:37 -0800 Subject: [PATCH 52/56] Add missing header required by std::accumulate --- src/wallet/rpcwallet.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 8a4ff1c1c..7c9c42ea5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -34,6 +34,8 @@ #include "json/json_spirit_value.h" #include "asyncrpcqueue.h" +#include + using namespace std; using namespace json_spirit; From 774489569ad3a092d2eba0ea1eaac5e90e9c9bbc Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 18 Jan 2017 16:54:39 -0800 Subject: [PATCH 53/56] Increase timeout for z_sendmany transaction in wallet.py qa test --- qa/rpc-tests/wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index cde7c1aad..e32561881 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -301,7 +301,7 @@ class WalletTest (BitcoinTestFramework): opids = [] opids.append(myopid) - timeout = 120 + timeout = 300 status = None for x in xrange(1, timeout): results = self.nodes[2].z_getoperationresult(opids) From 620c0e284567d851d69c1bf41c825a858619ebf9 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Thu, 19 Jan 2017 16:42:55 -0700 Subject: [PATCH 54/56] Add test for z_importkey rescanning from beginning of chain. --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/wallet_1941.py | 106 ++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100755 qa/rpc-tests/wallet_1941.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 821a637cd..28e59d554 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -15,6 +15,7 @@ testScripts=( 'wallet_protectcoinbase.py' 'wallet.py' 'wallet_nullifiers.py' + 'wallet_1941.py' 'listtransactions.py' 'mempool_resurrect_test.py' 'txn_doublespend.py' diff --git a/qa/rpc-tests/wallet_1941.py b/qa/rpc-tests/wallet_1941.py new file mode 100755 index 000000000..4958768f9 --- /dev/null +++ b/qa/rpc-tests/wallet_1941.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python2 +# Copyright (c) 2016 The Zcash developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# This is a regression test for #1941. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from time import * + +import sys + +starttime = 1388534400 + +class Wallet1941RegressionTest (BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 1) + + # Start nodes with -regtestprotectcoinbase to set fCoinbaseMustBeProtected to true. + def setup_network(self, split=False): + self.nodes = start_nodes(1, self.options.tmpdir, extra_args=[['-regtestprotectcoinbase','-debug=zrpc']] ) + self.is_network_split=False + + def add_second_node(self): + initialize_datadir(self.options.tmpdir, 1) + self.nodes.append(start_node(1, self.options.tmpdir, extra_args=['-regtestprotectcoinbase','-debug=zrpc'])) + self.nodes[1].setmocktime(starttime + 9000) + connect_nodes_bi(self.nodes,0,1) + self.sync_all() + + def wait_and_assert_operationid_status(self, myopid, in_status='success', in_errormsg=None): + print('waiting for async operation {}'.format(myopid)) + opids = [] + opids.append(myopid) + timeout = 300 + status = None + errormsg = None + for x in xrange(1, timeout): + results = self.nodes[0].z_getoperationresult(opids) + if len(results)==0: + sleep(1) + else: + status = results[0]["status"] + if status == "failed": + errormsg = results[0]['error']['message'] + break + print('...returned status: {}'.format(status)) + print('...error msg: {}'.format(errormsg)) + assert_equal(in_status, status) + if errormsg is not None: + assert(in_errormsg is not None) + assert_equal(in_errormsg in errormsg, True) + print('...returned error: {}'.format(errormsg)) + + def run_test (self): + print "Mining blocks..." + + self.nodes[0].setmocktime(starttime) + self.nodes[0].generate(101) + + mytaddr = self.nodes[0].getnewaddress() # where coins were mined + myzaddr = self.nodes[0].z_getnewaddress() + + # Send 10 coins to our zaddr. + recipients = [] + recipients.append({"address":myzaddr, "amount":Decimal('10.0') - Decimal('0.0001')}) + myopid = self.nodes[0].z_sendmany(mytaddr, recipients) + self.wait_and_assert_operationid_status(myopid) + self.nodes[0].generate(1) + + # Ensure the block times of the latest blocks exceed the variability + self.nodes[0].setmocktime(starttime + 3000) + self.nodes[0].generate(1) + self.nodes[0].setmocktime(starttime + 6000) + self.nodes[0].generate(1) + self.nodes[0].setmocktime(starttime + 9000) + self.nodes[0].generate(1) + + # Confirm the balance on node 0. + resp = self.nodes[0].z_getbalance(myzaddr) + assert_equal(Decimal(resp), Decimal('10.0') - Decimal('0.0001')) + + # Export the key for the zaddr from node 0. + key = self.nodes[0].z_exportkey(myzaddr) + + # Start the new wallet + self.add_second_node() + self.nodes[1].getnewaddress() + self.nodes[1].z_getnewaddress() + self.nodes[1].generate(101) + self.sync_all() + + # Import the key on node 1. + self.nodes[1].z_importkey(key) + + # Confirm that the balance on node 1 is valid now (node 1 must + # have rescanned) + resp = self.nodes[1].z_getbalance(myzaddr) + assert_equal(Decimal(resp), Decimal('10.0') - Decimal('0.0001')) + + +if __name__ == '__main__': + Wallet1941RegressionTest().main() From b8f737b364325b76c3e12cc4d94ea74768fa0f69 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Thu, 19 Jan 2017 19:21:32 -0700 Subject: [PATCH 55/56] Bump version to 1.0.5. --- README.md | 2 +- configure.ac | 2 +- contrib/debian/manpages/zcash-cli.1 | 4 ++-- contrib/debian/manpages/zcashd.1 | 4 ++-- contrib/gitian-descriptors/gitian-linux.yml | 2 +- src/clientversion.h | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c15d296e3..0dea83296 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Zcash 1.0.4 +Zcash 1.0.5 =========== What is Zcash? diff --git a/configure.ac b/configure.ac index bdcd3e00d..8a666ec6b 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 1) define(_CLIENT_VERSION_MINOR, 0) -define(_CLIENT_VERSION_REVISION, 4) +define(_CLIENT_VERSION_REVISION, 5) define(_CLIENT_VERSION_BUILD, 50) define(_ZC_BUILD_VAL, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, m4_incr(_CLIENT_VERSION_BUILD), m4_eval(_CLIENT_VERSION_BUILD < 50), 1, m4_eval(_CLIENT_VERSION_BUILD - 24), m4_eval(_CLIENT_VERSION_BUILD == 50), 1, , m4_eval(_CLIENT_VERSION_BUILD - 50))) define(_CLIENT_VERSION_SUFFIX, m4_if(m4_eval(_CLIENT_VERSION_BUILD < 25), 1, _CLIENT_VERSION_REVISION-beta$1, m4_eval(_CLIENT_VERSION_BUILD < 50), 1, _CLIENT_VERSION_REVISION-rc$1, m4_eval(_CLIENT_VERSION_BUILD == 50), 1, _CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION-$1))) diff --git a/contrib/debian/manpages/zcash-cli.1 b/contrib/debian/manpages/zcash-cli.1 index e5b47babd..bb0611eb7 100644 --- a/contrib/debian/manpages/zcash-cli.1 +++ b/contrib/debian/manpages/zcash-cli.1 @@ -1,9 +1,9 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH ZCASH-CLI "1" "December 2016" "Zcash RPC client version v1.0.4" "User Commands" +.TH ZCASH-CLI "1" "January 2017" "Zcash RPC client version v1.0.5" "User Commands" .SH NAME zcash-cli \- RPC client for the Zcash daemon .SH DESCRIPTION -Zcash RPC client version v1.0.4 +Zcash RPC client version v1.0.5 .SS "Usage:" .TP zcash\-cli [options] [params] diff --git a/contrib/debian/manpages/zcashd.1 b/contrib/debian/manpages/zcashd.1 index a110108be..eece9c6e5 100644 --- a/contrib/debian/manpages/zcashd.1 +++ b/contrib/debian/manpages/zcashd.1 @@ -1,9 +1,9 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.3. -.TH ZCASHD "1" "December 2016" "Zcash Daemon version v1.0.4" "User Commands" +.TH ZCASHD "1" "January 2017" "Zcash Daemon version v1.0.5" "User Commands" .SH NAME zcashd \- Network daemon for interacting with the Zcash blockchain .SH DESCRIPTION -Zcash Daemon version v1.0.4 +Zcash Daemon version v1.0.5 .SS "Usage:" .TP zcashd [options] diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 49c27537d..b5a7c2cb7 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "zcash-1.0.4" +name: "zcash-1.0.5" enable_cache: true distro: "debian" suites: diff --git a/src/clientversion.h b/src/clientversion.h index 6ca038285..33056403b 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -16,7 +16,7 @@ //! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it #define CLIENT_VERSION_MAJOR 1 #define CLIENT_VERSION_MINOR 0 -#define CLIENT_VERSION_REVISION 4 +#define CLIENT_VERSION_REVISION 5 #define CLIENT_VERSION_BUILD 50 //! Set to true for release, false for prerelease or test build From 5046d8cd055df8d36bea0d2e76c6c419846eba41 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Thu, 19 Jan 2017 19:25:00 -0700 Subject: [PATCH 56/56] Update release notes and Debian package. --- contrib/debian/changelog | 6 +++ doc/authors.md | 13 +++--- doc/release-notes/release-notes-1.0.5.md | 58 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 doc/release-notes/release-notes-1.0.5.md diff --git a/contrib/debian/changelog b/contrib/debian/changelog index eb9c5f415..ef0701804 100644 --- a/contrib/debian/changelog +++ b/contrib/debian/changelog @@ -1,3 +1,9 @@ +zcash (1.0.5) jessie; urgency=medium + + * 1.0.5 release. + + -- Zcash Company Thu, 19 Jan 2016 19:23:40 -0700 + zcash (1.0.4) jessie; urgency=medium * 1.0.4 release. diff --git a/doc/authors.md b/doc/authors.md index 2933964b5..c57a4631a 100644 --- a/doc/authors.md +++ b/doc/authors.md @@ -1,23 +1,23 @@ Zcash Contributors ================== -Jack Grigg (301) -Simon Liu (207) -Sean Bowe (176) +Jack Grigg (323) +Simon Liu (220) +Sean Bowe (180) Taylor Hornby (65) Daira Hopwood (62) Kevin Gallagher (38) -Jay Graber (34) +Jay Graber (35) +Wladimir J. van der Laan (10) Nathan Wilcox (10) -Wladimir J. van der Laan (9) Pieter Wuille (8) Cory Fields (7) +Paige Peterson (5) ITH4Coinomia (4) David Mercer (4) 4ZEC (4) lpescher (3) Patrick Strateman (3) -Paige Peterson (3) MarcoFalke (3) Alfie John (3) aniemerg (2) @@ -36,6 +36,7 @@ Lars-Magnus Skog (1) Jeffrey Walton (1) Gaurav Rana (1) Ethan Heilman (1) +Eran Tromer (1) Christian von Roques (1) Chirag Davé (1) Cameron Boehmer (1) diff --git a/doc/release-notes/release-notes-1.0.5.md b/doc/release-notes/release-notes-1.0.5.md new file mode 100644 index 000000000..7e2c6e1c8 --- /dev/null +++ b/doc/release-notes/release-notes-1.0.5.md @@ -0,0 +1,58 @@ +Eran Tromer (1): + CreateJoinSplit: add start_profiling() call + +Jack Grigg (22): + Extend createjoinsplit to benchmark parallel JoinSplits + Add total number of commitments to getblockchaininfo + Only enable getblocktemplate when wallet is enabled + Only run wallet tests when wallet is enabled + Add a tool for profiling the creation of JoinSplits + Exclude test binaries from make install + Scan the whole chain whenever a z-key is imported + Instruct users to run zcash-fetch-params if network params aren't available + Trigger metrics UI refresh on new messages + Strip out the SECURE flag in metrics UI so message style is detected + Handle newlines in UI messages + Suggest ./zcutil/fetch-params.sh as well + Update debug categories + Rename build-aux/m4/bitcoin_find_bdb48.m4 to remove version + Throw an error if zcash.conf is missing + Show a friendly message explaining why zcashd needs a zcash.conf + Fix gtest ordering broken by #1949 + Debian package lint + Generate Debian control file to fix shlibs lint + Create empty zcash.conf during performance measurements + Create empty zcash.conf during coverage checks + Coverage build system tweaks + +Jay Graber (1): + Update release process to check in with users who opened resolved issues + +Paige Peterson (2): + Create ISSUE_TEMPLATE.md + move template to subdirectory, fix typo, include prompt under describing issue section, include uploading file directly to github ticket as option for sharing logs + +Sean Bowe (4): + Add test for IncrementalMerkleTree::size(). + Add 'CreateJoinSplit' standalone utility to gitignore. + Add test for z_importkey rescanning from beginning of chain. + Bump version to 1.0.5. + +Simon Liu (13): + Fixes #1964 to catch general exception in z_sendmany and catch exceptions as reference-to-const. + Fixes #1967 by adding age of note to z_sendmany logging. + Fixes a bug where the unsigned transaction was logged by z_sendmany after a successful sign and send, meaning that the logged hash fragment would be different from the txid logged by "AddToWallet". This issue occured when sending from transparent addresses, as utxo inputs must be signed. It did not occur when sending from shielded addresses. + Bump COPYRIGHT_YEAR from 2016 to 2017. + Closes #1780. Result of z_getoperationstatus now sorted by creation time of operation + Remove UTF-8 BOM efbbbf from zcash.conf to avoid problems with command line tools + Closes #1097 so zcash-cli now displays license info like zcashd. + Fixes #1497 ZCA-009 by restricting data exporting to user defined folder. + Closes #1957 by adding tx serialization size to listtransactions output. + Fixes #1960: z_getoperationstatus/result now includes operation details. + Update walletbackup.py qa test to use -exportdir option + Add missing header required by std::accumulate + Increase timeout for z_sendmany transaction in wallet.py qa test + +Wladimir J. van der Laan (1): + rpc: Implement random-cookie based authentication +