From 5c519f9665abe4a8e4b1bc7617b22704602fb8b2 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Wed, 7 Aug 2019 10:15:35 -0400 Subject: [PATCH 01/22] Start teaching hushd to track various stats about shielded xtns --- src/chain.h | 28 ++++++++++++++++++++++++++++ src/main.cpp | 5 +++++ 2 files changed, 33 insertions(+) diff --git a/src/chain.h b/src/chain.h index b893e3a06..ce7ca7014 100644 --- a/src/chain.h +++ b/src/chain.h @@ -263,6 +263,29 @@ public: //! Change to 64-bit type when necessary; won't happen before 2030 unsigned int nChainTx; + //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. + //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as + //! inputs and outputs. + int64_t nShieldedChainTx; + + //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined + //! as a transaction containing JoinSplits and only shielded inputs and outputs + int64_t nFullyShieldedChainTx; + + //! (memory only) Number of deshielding transactions. A deshielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent output. + int64_t nDeshieldingChainTx; + + //! (memory only) Number of shielding transactions. A shielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent input + int64_t nShieldingChainTx; + + + //! (memory only) Number of shielding transactions which contain multiple shielded input notes + //! z_sendmany cannot make these and so it indicates raw shielded transaction usage currently + int64_t nMultipleShieldedInputsChainTx; + //! Verification status of this block. See enum BlockStatus unsigned int nStatus; @@ -321,6 +344,11 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; + nShieldedChainTx = 0; + nShieldingChainTx = 0; + nDeshieldingChainTx = 0; + nFullyShieldedChainTx = 0; + nMultipleShieldedInputsChainTx = 0; nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); diff --git a/src/main.cpp b/src/main.cpp index e01cba61f..cef96d381 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4734,6 +4734,11 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl while (!queue.empty()) { CBlockIndex *pindex = queue.front(); queue.pop_front(); + // TODO: Count zxtns properly, even amount=0 xtns + // TODO: count number of joinsplits instead + if(saplingValue>0) { + pindex->nShieldedChainTx = pindex->nShieldedChainTx ? 0 : pindex->nShieldedChainTx + 1; + } pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if (pindex->pprev) { if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { From bfe058f29b51270f01fff33d0915e75dceb25b08 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 8 Aug 2019 13:27:27 -0400 Subject: [PATCH 02/22] Keep track of more zxtn stats --- src/main.cpp | 51 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index cef96d381..d52d3a557 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4700,8 +4700,14 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; + pindexNew->nShieldedChainTx = 0; CAmount sproutValue = 0; CAmount saplingValue = 0; + int64_t nShieldedSpends=0,nShieldedOutputs=0; + int64_t nShieldedPayments=0; + int64_t nFullyShielded=0,nShielding=0,nDeshielding=0,nMultipleShieldedInputs=0; + bool hasShieldedTx = false; + for (auto tx : block.vtx) { // Negative valueBalance "takes" money from the transparent value pool // and adds it to the Sapling value pool. Positive valueBalance "gives" @@ -4713,6 +4719,41 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl sproutValue += js.vpub_old; sproutValue -= js.vpub_new; } + nShieldedSpends = tx.vShieldedSpend.size(); + nShieldedOutputs = tx.vShieldedOutput.size(); + + // If we have not seen any zxtns, see if current xtn has any + if(!hasShieldedTx) { + hasShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; + } + if(hasShieldedTx) { + if(tx.vin.size()==0 && tx.vout.size()==0) { + nFullyShielded++; + } + if(tx.vin.size()>0) { + nShielding++; + } + if(tx.out.size()>0) { + nDeshielding++; + } + if(nShieldedSpends>1) { + nMultipleShieldedInputs++; + } + //TODO: These are at best heuristics. Improve them as much as possible + //NOTE: You cannot compare payment data from different sets of heuristics + + if (nShieldedOutputs > 1) { + // If there are multiple shielded outputs, count each as a payment + nShieldedPayments += nShieldedOutputs; + } else { + nShieldedPayments++; + } + if (nShieldedSpends==1) { + // If we see a single zaddr input, that counts as a single shielded payment + nShieldedPayments++; + } + + } } pindexNew->nSproutValue = sproutValue; pindexNew->nChainSproutValue = boost::none; @@ -4734,10 +4775,12 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl while (!queue.empty()) { CBlockIndex *pindex = queue.front(); queue.pop_front(); - // TODO: Count zxtns properly, even amount=0 xtns - // TODO: count number of joinsplits instead - if(saplingValue>0) { - pindex->nShieldedChainTx = pindex->nShieldedChainTx ? 0 : pindex->nShieldedChainTx + 1; + + // Keep track of shielded transaction stats + if(hasShieldedTx) { + pindex->vShieldedSpend; + pindex->vShieldedOutput; + pindex->nShieldedChainTx = pindex->nShieldedChainTx ? 0 : pindex->nShieldedChainTx + 1; } pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if (pindex->pprev) { From 188b44937b5e654bb3bf04f1d831459e9af8a80e Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Thu, 8 Aug 2019 16:47:46 -0700 Subject: [PATCH 03/22] Clean up and improve some definitions of shielded stats; fix some compiler issues --- src/chain.h | 53 +++++++++++++++++++++++++++++++++++++++------------- src/main.cpp | 14 ++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/chain.h b/src/chain.h index ce7ca7014..077ad92d8 100644 --- a/src/chain.h +++ b/src/chain.h @@ -263,28 +263,51 @@ public: //! Change to 64-bit type when necessary; won't happen before 2030 unsigned int nChainTx; + //! (memory only) Number of payments (shielded or transparent) in the chain + //! up to and including this block. One transaction can contain one or more + //! payments. This stat allows us to calculate ratios of shielded/transparent + //! when combined with shielded payment stats + int64_t nChainPayments; + //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. - int64_t nShieldedChainTx; + int64_t nShieldedTx; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined - //! as a transaction containing JoinSplits and only shielded inputs and outputs - int64_t nFullyShieldedChainTx; + //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... + int64_t nFullyShieldedTx; + + //! (memory only) Number of shielding payments. A shielding payment is defined + //! as having a shielded output but transparent input: t->z + int64_t nShieldingPayments; + + //! (memory only) Number of shielded payments. A shielded payment is defined + //! as having a shielded input or output: t->z or z->t + int64_t nShieldedPayments; + + //! (memory only) Number of fully shielded payments. A fully shielded payment is defined + //! as having a shielded input and shielded output: z->z + int64_t nFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent output. - int64_t nDeshieldingChainTx; + int64_t nDeshieldingTx; + + //! (memory only) Number of deshielding payments. A deshielding payment is defined + //! as one transparent input and one shielded output: z->t + int64_t nDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent input - int64_t nShieldingChainTx; + // i.e. t->z or t->(z,t) or z->(z,z,t) + int64_t nShieldingTx; - - //! (memory only) Number of shielding transactions which contain multiple shielded input notes + //! (memory only) Number of shielded transactions which contain multiple shielded input notes //! z_sendmany cannot make these and so it indicates raw shielded transaction usage currently - int64_t nMultipleShieldedInputsChainTx; + int64_t nMultipleShieldedInputsTx; //! Verification status of this block. See enum BlockStatus unsigned int nStatus; @@ -344,11 +367,15 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; - nShieldedChainTx = 0; - nShieldingChainTx = 0; - nDeshieldingChainTx = 0; - nFullyShieldedChainTx = 0; - nMultipleShieldedInputsChainTx = 0; + nShieldedTx = 0; + nShieldingTx = 0; + nDeshieldingTx = 0; + nFullyShieldedTx = 0; + nShieldedPayments = 0; + nShieldingPayments = 0; + nDeshieldingPayments = 0; + nFullyShieldedPayments = 0; + nMultipleShieldedInputsTx = 0; nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); diff --git a/src/main.cpp b/src/main.cpp index 0eb2184d1..a3581e7f2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4769,7 +4769,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; - pindexNew->nShieldedChainTx = 0; + pindexNew->nChainPayments = 0; + pindexNew->nShieldedTx = 0; + pindexNew->nShieldingTx = 0; + pindexNew->nDeshieldingTx = 0; CAmount sproutValue = 0; CAmount saplingValue = 0; int64_t nShieldedSpends=0,nShieldedOutputs=0; @@ -4791,10 +4794,11 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedSpends = tx.vShieldedSpend.size(); nShieldedOutputs = tx.vShieldedOutput.size(); - // If we have not seen any zxtns, see if current xtn has any + // If we have not seen any zxtns, see if current block has any if(!hasShieldedTx) { hasShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; } + //TODO: this is if block has shielded txes, not if current xtn is shielded if(hasShieldedTx) { if(tx.vin.size()==0 && tx.vout.size()==0) { nFullyShielded++; @@ -4802,7 +4806,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl if(tx.vin.size()>0) { nShielding++; } - if(tx.out.size()>0) { + if(tx.vout.size()>0) { nDeshielding++; } if(nShieldedSpends>1) { @@ -4847,9 +4851,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // Keep track of shielded transaction stats if(hasShieldedTx) { - pindex->vShieldedSpend; - pindex->vShieldedOutput; - pindex->nShieldedChainTx = pindex->nShieldedChainTx ? 0 : pindex->nShieldedChainTx + 1; + pindex->nShieldedTx = pindex->nShieldedTx ? 0 : pindex->nShieldedTx + 1; } pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if (pindex->pprev) { From 8db846da675b47820a79a08fe6f7f495f3abbea8 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Thu, 8 Aug 2019 22:15:00 -0700 Subject: [PATCH 04/22] Improve calculation of shielded stats --- src/main.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index a3581e7f2..697df6cdf 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4778,8 +4778,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl int64_t nShieldedSpends=0,nShieldedOutputs=0; int64_t nShieldedPayments=0; int64_t nFullyShielded=0,nShielding=0,nDeshielding=0,nMultipleShieldedInputs=0; - bool hasShieldedTx = false; + bool hasShieldedTx = false; for (auto tx : block.vtx) { // Negative valueBalance "takes" money from the transparent value pool // and adds it to the Sapling value pool. Positive valueBalance "gives" @@ -4794,11 +4794,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedSpends = tx.vShieldedSpend.size(); nShieldedOutputs = tx.vShieldedOutput.size(); - // If we have not seen any zxtns, see if current block has any - if(!hasShieldedTx) { - hasShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; - } - //TODO: this is if block has shielded txes, not if current xtn is shielded + hasShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; + if(hasShieldedTx) { if(tx.vin.size()==0 && tx.vout.size()==0) { nFullyShielded++; @@ -4813,16 +4810,21 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nMultipleShieldedInputs++; } //TODO: These are at best heuristics. Improve them as much as possible - //NOTE: You cannot compare payment data from different sets of heuristics + //TODO: Take into account change addresses + //NOTE: You cannot compare stats generated from different sets of heuristics - if (nShieldedOutputs > 1) { - // If there are multiple shielded outputs, count each as a payment - nShieldedPayments += nShieldedOutputs; - } else { - nShieldedPayments++; - } - if (nShieldedSpends==1) { - // If we see a single zaddr input, that counts as a single shielded payment + if (nShieldedOutputs >= 1) { + // If there are shielded outputs, count each as a payment + // t->(t,t,z) = 1 shielded payment + // z->(z,z) = 1 shielded payment + shielded change + // t->(z,z) = 1 shielded payment + shielded change + // t->(t,z) = 1 shielded payment + transparent change + // z->(z,z,z) = 2 shielded payments + shielded change + nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + } else if (nShieldedSpends >=1) { + // z->t = 1 shielded payment + // (z,z)->z = 1 shielded payment + // z->(t,t) = 1 shielded payment nShieldedPayments++; } From 6ddcdc249757c5bc1e0650ffe979f4cd79ef20d6 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 10 Aug 2019 22:00:14 -0700 Subject: [PATCH 05/22] Various improvements to shielded transaction and payment stats gathering --- src/main.cpp | 81 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 32 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 697df6cdf..450a60dd5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4767,19 +4767,20 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) { - pindexNew->nTx = block.vtx.size(); - pindexNew->nChainTx = 0; + pindexNew->nTx = block.vtx.size(); + pindexNew->nChainTx = 0; pindexNew->nChainPayments = 0; - pindexNew->nShieldedTx = 0; - pindexNew->nShieldingTx = 0; + pindexNew->nShieldedTx = 0; + pindexNew->nShieldingTx = 0; pindexNew->nDeshieldingTx = 0; - CAmount sproutValue = 0; - CAmount saplingValue = 0; - int64_t nShieldedSpends=0,nShieldedOutputs=0; - int64_t nShieldedPayments=0; - int64_t nFullyShielded=0,nShielding=0,nDeshielding=0,nMultipleShieldedInputs=0; + CAmount sproutValue = 0; + CAmount saplingValue = 0; + bool isShieldedTx = false; + + int64_t nShieldedSpends=0,nShieldedOutputs=0,nChainPayments=0; + int64_t nShieldedTx=0,nFullyShieldedTx=0,nDeshieldingTx=0,nShieldingTx=0; + int64_t nShieldedPayments=0,nFullyShieldedPayments=0,nShieldingPayments=0,nDeshieldingPayments=0; - bool hasShieldedTx = false; for (auto tx : block.vtx) { // Negative valueBalance "takes" money from the transparent value pool // and adds it to the Sapling value pool. Positive valueBalance "gives" @@ -4791,43 +4792,49 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl sproutValue += js.vpub_old; sproutValue -= js.vpub_new; } - nShieldedSpends = tx.vShieldedSpend.size(); - nShieldedOutputs = tx.vShieldedOutput.size(); + nShieldedSpends = tx.vShieldedSpend.size(); + nShieldedOutputs = tx.vShieldedOutput.size(); + isShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; - hasShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; - - if(hasShieldedTx) { + if(isShieldedTx) { + nShieldedTx++; if(tx.vin.size()==0 && tx.vout.size()==0) { - nFullyShielded++; + nFullyShieldedTx++; } if(tx.vin.size()>0) { - nShielding++; + nShieldingTx++; } if(tx.vout.size()>0) { - nDeshielding++; + nDeshieldingTx++; } - if(nShieldedSpends>1) { - nMultipleShieldedInputs++; - } - //TODO: These are at best heuristics. Improve them as much as possible - //TODO: Take into account change addresses - //NOTE: You cannot compare stats generated from different sets of heuristics + //NOTE: These are at best heuristics. Improve them as much as possible + // You cannot compare stats generated from different sets of heuristics if (nShieldedOutputs >= 1) { // If there are shielded outputs, count each as a payment + // By default, if there is more than 1 output, we assume 1 change output which is not a payment. + // In the case of multiple outputs which spend inputs exactly, there is no change output and this + // heuristic will undercount payments. Since this edge case is rare, this seems acceptable. // t->(t,t,z) = 1 shielded payment // z->(z,z) = 1 shielded payment + shielded change // t->(z,z) = 1 shielded payment + shielded change // t->(t,z) = 1 shielded payment + transparent change + // (z,z)->z = 1 shielded payment (has this xtn ever occurred?) // z->(z,z,z) = 2 shielded payments + shielded change + // Assume that there is always 1 change output when there are more than one nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; } else if (nShieldedSpends >=1) { + // Shielded inputs with no shielded outputs. We know none are change output because + // change would flow back to the zaddr // z->t = 1 shielded payment - // (z,z)->z = 1 shielded payment - // z->(t,t) = 1 shielded payment - nShieldedPayments++; + // z->(t,t) = 2 shielded payments + // z->(t,t,t) = 3 shielded payments + nShieldedPayments += tx.vout.size(); } - + nChainPayments += nShieldedPayments; + } else { + // No shielded payments, add transparent payments minus a change address + nChainPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); } } pindexNew->nSproutValue = sproutValue; @@ -4839,6 +4846,18 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); + + // Store data for later use by gettxchainstats and other RPCs + pindexNew->nChainPayments = nChainPayments; + pindexNew->nShieldedTx = nShieldedTx; + pindexNew->nFullyShieldedTx = nFullyShieldedTx; + pindexNew->nDeshieldingTx = nDeshieldingTx; + pindexNew->nShieldingTx = nShieldingTx; + pindexNew->nShieldedPayments = nShieldedPayments; + pindexNew->nFullyShieldedPayments = nFullyShieldedPayments; + pindexNew->nDeshieldingPayments = nDeshieldingPayments; + pindexNew->nShieldingPayments = nShieldingPayments; + setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { @@ -4851,10 +4870,6 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); - // Keep track of shielded transaction stats - if(hasShieldedTx) { - pindex->nShieldedTx = pindex->nShieldedTx ? 0 : pindex->nShieldedTx + 1; - } pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if (pindex->pprev) { if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { @@ -4892,6 +4907,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } } + fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, FullyShieldedTx=%d\n", + pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nFullyShieldedTx ); return true; } From 464251bdc3590f9ddeb7b0e906b452f7982a570e Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Mon, 12 Aug 2019 17:38:37 -0700 Subject: [PATCH 06/22] Add total payment stats to getchaintxstats --- src/rpc/blockchain.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index c63def984..374725ba5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1949,7 +1949,7 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) "{\n" " \"time\": xxxxx, (numeric) The timestamp for the final block in the window in UNIX format.\n" " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" - " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" + " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" " \"window_block_count\": xxxxx, (numeric) Size of the window in number of blocks.\n" " \"window_tx_count\": xxxxx, (numeric) The number of transactions in the window. Only returned if \"window_block_count\" is > 0.\n" " \"window_interval\": xxxxx, (numeric) The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0.\n" @@ -1991,8 +1991,9 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) } const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); - int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); - int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); @@ -2003,7 +2004,8 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("window_tx_count", nTxDiff); ret.pushKV("window_interval", nTimeDiff); if (nTimeDiff > 0) { - ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); } } From 70eb669e6382957c50795a388f0675e31ba44026 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 13 Aug 2019 07:08:31 -0700 Subject: [PATCH 07/22] Return json data about shielded xtns/payments --- src/rpc/blockchain.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 374725ba5..5803f8a7f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1994,18 +1994,25 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; + int64_t nShieldedPaymentsDiff = pindex->nShieldedPayments - pindexPast->nShieldedPayments; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); ret.pushKV("txcount", (int64_t)pindex->nChainTx); + ret.pushKV("shielded_txcount", (int64_t)pindex->nShieldedTx); + ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nFullyShieldedTx); + ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); ret.pushKV("window_block_count", blockcount); if (blockcount > 0) { ret.pushKV("window_tx_count", nTxDiff); + ret.pushKV("window_payment_count", nPaymentsDiff); + ret.pushKV("window_shielded_payment_count", nShieldedPaymentsDiff); ret.pushKV("window_interval", nTimeDiff); if (nTimeDiff > 0) { ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); + ret.pushKV("shieled_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); } } From 22c0a2292659813633a9df7a8e41ca4f670119e9 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 05:56:07 -0700 Subject: [PATCH 08/22] Compiles, RPC gives back some shielded stats, which have bugs --- src/chain.h | 76 +++++++++++++++++++++++++++++++++++------- src/main.cpp | 27 +++++++++------ src/rpc/blockchain.cpp | 5 ++- 3 files changed, 83 insertions(+), 25 deletions(-) diff --git a/src/chain.h b/src/chain.h index 077ad92d8..52028563d 100644 --- a/src/chain.h +++ b/src/chain.h @@ -254,22 +254,18 @@ public: //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block CChainPower chainPower; + // block stats //! Number of transactions in this block. //! Note: in a potential headers-first mode, this number cannot be relied upon unsigned int nTx; - //! (memory only) Number of transactions in the chain up to and including this block. - //! This value will be non-zero only if and only if transactions for this block and all its parents are available. - //! Change to 64-bit type when necessary; won't happen before 2030 - unsigned int nChainTx; - - //! (memory only) Number of payments (shielded or transparent) in the chain + //! (memory only) Number of payments (shielded or transparent) in the block //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent //! when combined with shielded payment stats - int64_t nChainPayments; + int64_t nPayments; - //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. + //! (memory only) Number of shielded transactions (of any kind) in the block up to and including this block. //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. @@ -305,9 +301,56 @@ public: // i.e. t->z or t->(z,t) or z->(z,z,t) int64_t nShieldingTx; - //! (memory only) Number of shielded transactions which contain multiple shielded input notes - //! z_sendmany cannot make these and so it indicates raw shielded transaction usage currently - int64_t nMultipleShieldedInputsTx; + + + // chain stats + //! (memory only) Number of transactions in the chain up to and including this block. + //! This value will be non-zero only if and only if transactions for this block and all its parents are available. + //! Change to 64-bit type when necessary; won't happen before 2030 + unsigned int nChainTx; + + //! (memory only) Number of payments (shielded or transparent) in the chain + //! up to and including this block. One transaction can contain one or more + //! payments. This stat allows us to calculate ratios of shielded/transparent + //! when combined with shielded payment stats + int64_t nChainPayments; + + //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. + //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as + //! inputs and outputs. + int64_t nChainShieldedTx; + + //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined + //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... + int64_t nChainFullyShieldedTx; + + //! (memory only) Number of shielding payments. A shielding payment is defined + //! as having a shielded output but transparent input: t->z + int64_t nChainShieldingPayments; + + //! (memory only) Number of shielded payments. A shielded payment is defined + //! as having a shielded input or output: t->z or z->t + int64_t nChainShieldedPayments; + + //! (memory only) Number of fully shielded payments. A fully shielded payment is defined + //! as having a shielded input and shielded output: z->z + int64_t nChainFullyShieldedPayments; + + //! (memory only) Number of deshielding transactions. A deshielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent output. + int64_t nChainDeshieldingTx; + + //! (memory only) Number of deshielding payments. A deshielding payment is defined + //! as one transparent input and one shielded output: z->t + int64_t nChainDeshieldingPayments; + + //! (memory only) Number of shielding transactions. A shielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent input + // i.e. t->z or t->(z,t) or z->(z,z,t) + int64_t nChainShieldingTx; + //! Verification status of this block. See enum BlockStatus unsigned int nStatus; @@ -367,6 +410,15 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; + + nChainShieldedTx = 0; + nChainShieldingTx = 0; + nChainDeshieldingTx = 0; + nChainFullyShieldedTx = 0; + nChainShieldedPayments = 0; + nChainShieldingPayments = 0; + nChainDeshieldingPayments = 0; + nChainFullyShieldedPayments = 0; nShieldedTx = 0; nShieldingTx = 0; nDeshieldingTx = 0; @@ -375,7 +427,7 @@ public: nShieldingPayments = 0; nDeshieldingPayments = 0; nFullyShieldedPayments = 0; - nMultipleShieldedInputsTx = 0; + nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); diff --git a/src/main.cpp b/src/main.cpp index 450a60dd5..df3eaa2a5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4770,14 +4770,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; pindexNew->nChainPayments = 0; - pindexNew->nShieldedTx = 0; - pindexNew->nShieldingTx = 0; - pindexNew->nDeshieldingTx = 0; CAmount sproutValue = 0; CAmount saplingValue = 0; bool isShieldedTx = false; - - int64_t nShieldedSpends=0,nShieldedOutputs=0,nChainPayments=0; + int64_t nShieldedSpends=0,nShieldedOutputs=0,nPayments=0; int64_t nShieldedTx=0,nFullyShieldedTx=0,nDeshieldingTx=0,nShieldingTx=0; int64_t nShieldedPayments=0,nFullyShieldedPayments=0,nShieldingPayments=0,nDeshieldingPayments=0; @@ -4831,12 +4827,14 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // z->(t,t,t) = 3 shielded payments nShieldedPayments += tx.vout.size(); } - nChainPayments += nShieldedPayments; + //TODO: correctly add shielded payments to total chain payments + nPayments += nShieldedPayments; } else { // No shielded payments, add transparent payments minus a change address - nChainPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); + nPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); } } + pindexNew->nSproutValue = sproutValue; pindexNew->nChainSproutValue = boost::none; pindexNew->nSaplingValue = saplingValue; @@ -4847,8 +4845,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - // Store data for later use by gettxchainstats and other RPCs - pindexNew->nChainPayments = nChainPayments; + pindexNew->nPayments = nPayments; pindexNew->nShieldedTx = nShieldedTx; pindexNew->nFullyShieldedTx = nFullyShieldedTx; pindexNew->nDeshieldingTx = nDeshieldingTx; @@ -4870,7 +4867,17 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; + pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; + pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; + pindex->nChainDeshieldingTx = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx; + + pindex->nChainShieldedPayments = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments; + pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; + pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; + pindex->nChainDeshieldingPayments = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments; + if (pindex->pprev) { if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 5803f8a7f..dea832d97 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1994,14 +1994,13 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; - int64_t nShieldedPaymentsDiff = pindex->nShieldedPayments - pindexPast->nShieldedPayments; + int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); ret.pushKV("txcount", (int64_t)pindex->nChainTx); - ret.pushKV("shielded_txcount", (int64_t)pindex->nShieldedTx); + ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nFullyShieldedTx); - ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); ret.pushKV("window_block_count", blockcount); if (blockcount > 0) { From c9b00ef093723cdce5730a2cbaf1de1e6c94cc3f Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 06:15:03 -0700 Subject: [PATCH 09/22] Add chain payments --- src/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.cpp b/src/main.cpp index df3eaa2a5..d432565a7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4873,6 +4873,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; pindex->nChainDeshieldingTx = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx; + pindex->nChainPayments = (pindex->pprev ? pindex->pprev->nChainPayments : 0) + pindex->nPayments; pindex->nChainShieldedPayments = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments; pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; From 1a6c19dc811624af3d2ccd5865a68784f6719be0 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 06:26:13 -0700 Subject: [PATCH 10/22] Also calc shielded stats on index load --- src/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index d432565a7..8b37525fc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6147,7 +6147,11 @@ bool static LoadBlockIndexDB() if (pindex->nTx > 0) { if (pindex->pprev) { if (pindex->pprev->nChainTx) { - pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; } else { From 2b3023444d53d84639aeefb486c2dbe3053b7aa0 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 12:46:40 -0700 Subject: [PATCH 11/22] Properly serialize/deserialize new blockindex with zstats --- src/chain.h | 11 +++++++++++ src/main.cpp | 39 ++++++++++++++++++++++++++++++--------- src/txdb.cpp | 43 ++++++++++++++++++++++++++----------------- 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/src/chain.h b/src/chain.h index 52028563d..f815b5c8a 100644 --- a/src/chain.h +++ b/src/chain.h @@ -613,6 +613,17 @@ public: READWRITE(VARINT(chainPower.nHeight)); READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); + + READWRITE(VARINT(nShieldedTx)); + READWRITE(VARINT(nShieldingTx)); + READWRITE(VARINT(nDeshieldingTx)); + READWRITE(VARINT(nFullyShieldedTx)); + + READWRITE(VARINT(nShieldedPayments)); + READWRITE(VARINT(nShieldingPayments)); + READWRITE(VARINT(nDeshieldingPayments)); + READWRITE(VARINT(nFullyShieldedPayments)); + if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(nFile)); if (nStatus & BLOCK_HAVE_DATA) diff --git a/src/main.cpp b/src/main.cpp index 8b37525fc..616054880 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6148,9 +6148,15 @@ bool static LoadBlockIndexDB() if (pindex->pprev) { if (pindex->pprev->nChainTx) { - pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; - pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; - pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx; + pindex->nChainShieldingPayments = pindex->pprev->nChainShieldingPayments + pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainDeshieldingPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainFullyShieldedTx = pindex->pprev->nChainFullyShieldedTx + pindex->nFullyShieldedTx; + pindex->nChainFullyShieldedPayments = pindex->pprev->nChainFullyShieldedPayments + pindex->nFullyShieldedPayments; if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; @@ -6163,15 +6169,30 @@ bool static LoadBlockIndexDB() pindex->nChainSaplingValue = boost::none; } } else { - pindex->nChainTx = 0; - pindex->nChainSproutValue = boost::none; - pindex->nChainSaplingValue = boost::none; + pindex->nChainTx = 0; + pindex->nChainShieldedTx = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainShieldedPayments = 0; + pindex->nChainShieldingPayments = 0; + pindex->nChainDeshieldingTx = 0; + pindex->nChainDeshieldingPayments = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainFullyShieldedPayments = 0; + pindex->nChainSproutValue = boost::none; + pindex->nChainSaplingValue = boost::none; mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); } } else { - pindex->nChainTx = pindex->nTx; - pindex->nChainSproutValue = pindex->nSproutValue; - pindex->nChainSaplingValue = pindex->nSaplingValue; + pindex->nChainTx = pindex->nTx; + pindex->nChainShieldedTx = pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->nShieldingTx; + pindex->nChainShieldingPayments = pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->nDeshieldingTx; + pindex->nChainDeshieldingPayments = pindex->nDeshieldingPayments; + pindex->nChainFullyShieldedPayments = pindex->nFullyShieldedPayments; + pindex->nChainSproutValue = pindex->nSproutValue; + pindex->nChainSaplingValue = pindex->nSaplingValue; } } // Construct in-memory chain of branch IDs. diff --git a/src/txdb.cpp b/src/txdb.cpp index 82885044e..a462061a1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -702,24 +702,33 @@ bool CBlockTreeDB::LoadBlockIndexGuts() CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->SetHeight(diskindex.GetHeight()); - pindexNew->nFile = diskindex.nFile; - pindexNew->nDataPos = diskindex.nDataPos; - pindexNew->nUndoPos = diskindex.nUndoPos; - pindexNew->hashSproutAnchor = diskindex.hashSproutAnchor; - pindexNew->nVersion = diskindex.nVersion; - pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; + pindexNew->nFile = diskindex.nFile; + pindexNew->nDataPos = diskindex.nDataPos; + pindexNew->nUndoPos = diskindex.nUndoPos; + pindexNew->hashSproutAnchor = diskindex.hashSproutAnchor; + pindexNew->nVersion = diskindex.nVersion; + pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->hashFinalSaplingRoot = diskindex.hashFinalSaplingRoot; - pindexNew->nTime = diskindex.nTime; - pindexNew->nBits = diskindex.nBits; - pindexNew->nNonce = diskindex.nNonce; - pindexNew->nSolution = diskindex.nSolution; - pindexNew->nStatus = diskindex.nStatus; - pindexNew->nCachedBranchId = diskindex.nCachedBranchId; - pindexNew->nTx = diskindex.nTx; - pindexNew->nSproutValue = diskindex.nSproutValue; - pindexNew->nSaplingValue = diskindex.nSaplingValue; - pindexNew->segid = diskindex.segid; - pindexNew->nNotaryPay = diskindex.nNotaryPay; + pindexNew->nTime = diskindex.nTime; + pindexNew->nBits = diskindex.nBits; + pindexNew->nNonce = diskindex.nNonce; + pindexNew->nSolution = diskindex.nSolution; + pindexNew->nStatus = diskindex.nStatus; + pindexNew->nCachedBranchId = diskindex.nCachedBranchId; + pindexNew->nTx = diskindex.nTx; + pindexNew->nShieldedTx = diskindex.nShieldedTx; + pindexNew->nShieldedPayments = diskindex.nShieldedPayments; + pindexNew->nShieldingTx = diskindex.nShieldingTx; + pindexNew->nShieldingPayments = diskindex.nShieldingPayments; + pindexNew->nDeshieldingTx = diskindex.nDeshieldingTx; + pindexNew->nDeshieldingPayments = diskindex.nDeshieldingPayments; + pindexNew->nFullyShieldedTx = diskindex.nFullyShieldedTx; + pindexNew->nFullyShieldedPayments = diskindex.nFullyShieldedPayments; + pindexNew->nSproutValue = diskindex.nSproutValue; + pindexNew->nSaplingValue = diskindex.nSaplingValue; + pindexNew->segid = diskindex.segid; + pindexNew->nNotaryPay = diskindex.nNotaryPay; + //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks auto header = pindexNew->GetBlockHeader(); From 469c3c9cf27670f5b78b6229889da3f49462c533 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 21:20:10 -0700 Subject: [PATCH 12/22] So many zstats --- src/rpc/blockchain.cpp | 72 +++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index dea832d97..436f3d613 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1990,29 +1990,77 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) } } - const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); - int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); - int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; - int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; - int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; + int64_t nShieldedTxDiff = pindex->nChainShieldedTx - pindexPast->nChainShieldedTx; + int64_t nShieldingTxDiff = pindex->nChainShieldingTx - pindexPast->nChainShieldingTx; + int64_t nDeshieldingTxDiff = pindex->nChainDeshieldingTx - pindexPast->nChainDeshieldingTx; + int64_t nFullyShieldedTxDiff = pindex->nChainFullyShieldedTx - pindexPast->nChainFullyShieldedTx; + int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; + int64_t nShieldingPaymentsDiff = pindex->nChainShieldingPayments - pindexPast->nChainShieldingPayments; + int64_t nDeshieldingPaymentsDiff = pindex->nChainDeshieldingPayments - pindexPast->nChainDeshieldingPayments; + int64_t nFullyShieldedPaymentsDiff = pindex->nChainFullyShieldedPayments - pindexPast->nChainFullyShieldedPayments; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); + ret.pushKV("txcount", (int64_t)pindex->nChainTx); ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); - ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nFullyShieldedTx); + ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nChainFullyShieldedTx); + ret.pushKV("deshielding_txcount", (int64_t)pindex->nChainDeshieldingTx); + ret.pushKV("shielding_txcount", (int64_t)pindex->nChainShieldingTx); + ret.pushKV("shielded_payments", (int64_t)pindex->nChainShieldedPayments); + ret.pushKV("fully_shielded_payments", (int64_t)pindex->nChainFullyShieldedPayments); + ret.pushKV("deshielding_payments", (int64_t)pindex->nChainDeshieldingPayments); + ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments); + ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); ret.pushKV("window_block_count", blockcount); + if (blockcount > 0) { - ret.pushKV("window_tx_count", nTxDiff); - ret.pushKV("window_payment_count", nPaymentsDiff); - ret.pushKV("window_shielded_payment_count", nShieldedPaymentsDiff); ret.pushKV("window_interval", nTimeDiff); + ret.pushKV("window_txcount", nTxDiff); + ret.pushKV("window_payments", nPaymentsDiff); + ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); + ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); + ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); + ret.pushKV("window_deshielding_payments", nDeshieldingPaymentsDiff); + if (nTimeDiff > 0) { - ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); - ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); - ret.pushKV("shieled_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); + ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + ret.pushKV("shielded_txrate", ((double)nShieldedTxDiff) / nTimeDiff); + ret.pushKV("fully_shielded_txrate", ((double)nFullyShieldedTxDiff) / nTimeDiff); + ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); + ret.pushKV("shielded_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); + ret.pushKV("fully_shielded_paymentrate", ((double)nFullyShieldedPaymentsDiff) / nTimeDiff); } + if (nTxDiff > 0) { + ret.pushKV("shielded_tx_percent", ((double)pindex->nShieldedTx) / nTxDiff); + ret.pushKV("fully_shielded_tx_percent", ((double)pindex->nFullyShieldedTx) / nTxDiff); + ret.pushKV("shielding_tx_percent", ((double)pindex->nShieldingTx) / nTxDiff); + ret.pushKV("deshielding_tx_percent", ((double)pindex->nDeshieldingTx) / nTxDiff); + } + if (nPaymentsDiff > 0) { + ret.pushKV("shielded_payments_percent", ((double)pindex->nShieldedPayments) / nPaymentsDiff); + ret.pushKV("fully_shielded_payments_percent", ((double)pindex->nFullyShieldedPayments) / nPaymentsDiff); + ret.pushKV("shielding_payments_percent", ((double)pindex->nShieldingPayments) / nPaymentsDiff); + ret.pushKV("deshielding_payments_percent", ((double)pindex->nDeshieldingPayments) / nPaymentsDiff); + } + // Shielded-only statistics + UniValue shielded(UniValue::VOBJ); + if (pindex->nShieldedTx > 0 ) { + shielded.pushKV("fully_shielded_tx_percent", ((double)pindex->nFullyShieldedTx) / pindex->nShieldedTx ); + shielded.pushKV("shielding_tx_percent", ((double)pindex->nShieldingTx) / pindex->nShieldedTx ); + shielded.pushKV("deshielding_tx_percent", ((double)pindex->nDeshieldingTx) / pindex->nShieldedTx ); + } + if (pindex->nShieldedPayments > 0 ) { + shielded.pushKV("fully_shielded_payments_percent", ((double)pindex->nFullyShieldedPayments) / pindex->nShieldedPayments ); + shielded.pushKV("shielding_payments_percent", ((double)pindex->nShieldingPayments) / pindex->nShieldedPayments ); + shielded.pushKV("deshielding_payments_percent", ((double)pindex->nDeshieldingPayments) / pindex->nShieldedPayments ); + } + ret.pushKV("shielded_only", shielded); } return ret; From 80b1479184d1ecf8d7fcbd1cdca95a79ad4db55b Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 21:54:48 -0700 Subject: [PATCH 13/22] Improve shielded-only zstats --- src/rpc/blockchain.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 436f3d613..f963052d4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2048,19 +2048,21 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("shielding_payments_percent", ((double)pindex->nShieldingPayments) / nPaymentsDiff); ret.pushKV("deshielding_payments_percent", ((double)pindex->nDeshieldingPayments) / nPaymentsDiff); } + // Shielded-only statistics UniValue shielded(UniValue::VOBJ); - if (pindex->nShieldedTx > 0 ) { - shielded.pushKV("fully_shielded_tx_percent", ((double)pindex->nFullyShieldedTx) / pindex->nShieldedTx ); - shielded.pushKV("shielding_tx_percent", ((double)pindex->nShieldingTx) / pindex->nShieldedTx ); - shielded.pushKV("deshielding_tx_percent", ((double)pindex->nDeshieldingTx) / pindex->nShieldedTx ); + if (nShieldedTxDiff > 0) { + shielded.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nShieldedTxDiff ); + shielded.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nShieldedTxDiff ); + shielded.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nShieldedTxDiff ); } - if (pindex->nShieldedPayments > 0 ) { - shielded.pushKV("fully_shielded_payments_percent", ((double)pindex->nFullyShieldedPayments) / pindex->nShieldedPayments ); - shielded.pushKV("shielding_payments_percent", ((double)pindex->nShieldingPayments) / pindex->nShieldedPayments ); - shielded.pushKV("deshielding_payments_percent", ((double)pindex->nDeshieldingPayments) / pindex->nShieldedPayments ); + if (nShieldedPaymentsDiff > 0) { + shielded.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nShieldedPaymentsDiff ); } - ret.pushKV("shielded_only", shielded); + if(nShieldedTxDiff+nShieldedPaymentsDiff > 0) + ret.pushKV("shielded_only", shielded); } return ret; From 3f94feb8369c15d3b60d5006cc0d1835bbacb726 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sat, 17 Aug 2019 22:08:17 -0700 Subject: [PATCH 14/22] Shielding+deshielding payments in z-only section --- src/main.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 616054880..bcfb83f77 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4818,14 +4818,17 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // (z,z)->z = 1 shielded payment (has this xtn ever occurred?) // z->(z,z,z) = 2 shielded payments + shielded change // Assume that there is always 1 change output when there are more than one - nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + // TODO: fully shielded should not count toward shielding/deshielding + nShieldingPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; } else if (nShieldedSpends >=1) { // Shielded inputs with no shielded outputs. We know none are change output because // change would flow back to the zaddr // z->t = 1 shielded payment // z->(t,t) = 2 shielded payments // z->(t,t,t) = 3 shielded payments - nShieldedPayments += tx.vout.size(); + nShieldedPayments += tx.vout.size(); + nDeshieldingPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); } //TODO: correctly add shielded payments to total chain payments nPayments += nShieldedPayments; From 5365d15e693f1a3e1e5543ca48ce19d6490157f7 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sun, 18 Aug 2019 18:06:12 -0700 Subject: [PATCH 15/22] Correct various stats and add block height to response --- src/main.cpp | 15 +++++++++------ src/rpc/blockchain.cpp | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index bcfb83f77..fda64ce02 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4796,11 +4796,9 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedTx++; if(tx.vin.size()==0 && tx.vout.size()==0) { nFullyShieldedTx++; - } - if(tx.vin.size()>0) { + } else if(tx.vin.size()>0) { nShieldingTx++; - } - if(tx.vout.size()>0) { + } else if(tx.vout.size()>0) { nDeshieldingTx++; } //NOTE: These are at best heuristics. Improve them as much as possible @@ -4819,8 +4817,13 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // z->(z,z,z) = 2 shielded payments + shielded change // Assume that there is always 1 change output when there are more than one nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; - // TODO: fully shielded should not count toward shielding/deshielding - nShieldingPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + + // Fully shielded do not count toward shielding/deshielding + if(tx.vin.size()==0 && tx.vout.size()==0) { + nFullyShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + } else { + nShieldingPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + } } else if (nShieldedSpends >=1) { // Shielded inputs with no shielded outputs. We know none are change output because // change would flow back to the zaddr diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f963052d4..d18f4e5c1 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2017,6 +2017,7 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments); ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); + ret.pushKV("window_final_block_height", pindex->GetHeight()); ret.pushKV("window_block_count", blockcount); if (blockcount > 0) { From 5c310aa0486ee910fb9bc8f3d00977c8e17079c5 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Sun, 18 Aug 2019 18:55:34 -0700 Subject: [PATCH 16/22] Define -zindex --- src/chain.h | 21 +++++++++++++-------- src/init.cpp | 6 ++++++ src/main.cpp | 15 ++++++++++++--- src/main.h | 1 + 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/chain.h b/src/chain.h index f815b5c8a..1a0a7057d 100644 --- a/src/chain.h +++ b/src/chain.h @@ -29,6 +29,7 @@ class CChainPower; #include "tinyformat.h" #include "uint256.h" extern int8_t is_STAKED(const char *chain_name); +extern bool fZindex; #include @@ -614,15 +615,19 @@ public: READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); - READWRITE(VARINT(nShieldedTx)); - READWRITE(VARINT(nShieldingTx)); - READWRITE(VARINT(nDeshieldingTx)); - READWRITE(VARINT(nFullyShieldedTx)); + // These values only serialized when -zindex=1 + //if (fZindex != GetBoolArg("-zindex", false)) { + if(fZindex) { + READWRITE(VARINT(nShieldedTx)); + READWRITE(VARINT(nShieldingTx)); + READWRITE(VARINT(nDeshieldingTx)); + READWRITE(VARINT(nFullyShieldedTx)); - READWRITE(VARINT(nShieldedPayments)); - READWRITE(VARINT(nShieldingPayments)); - READWRITE(VARINT(nDeshieldingPayments)); - READWRITE(VARINT(nFullyShieldedPayments)); + READWRITE(VARINT(nShieldedPayments)); + READWRITE(VARINT(nShieldingPayments)); + READWRITE(VARINT(nDeshieldingPayments)); + READWRITE(VARINT(nFullyShieldedPayments)); + } if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(nFile)); diff --git a/src/init.cpp b/src/init.cpp index f8875af44..18985e842 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1668,6 +1668,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) break; } + // Check for changed -zindex state + if (fZindex != GetBoolArg("-zindex", false)) { + strLoadError = _("You need to rebuild the database using -reindex to change -zindex"); + break; + } + // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. if (fHavePruned && !fPruneMode) { diff --git a/src/main.cpp b/src/main.cpp index fda64ce02..f1057699f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -98,6 +98,7 @@ bool fExperimentalMode = true; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fZindex = false; bool fAddressIndex = false; bool fTimestampIndex = false; bool fSpentIndex = false; @@ -6279,6 +6280,9 @@ bool static LoadBlockIndexDB() // Check whether we have an address index pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a shielded index + pblocktree->ReadFlag("zindex", fZindex); + LogPrintf("%s: shielded index %s\n", __func__, fZindex ? "enabled" : "disabled"); // Check whether we have a timestamp index pblocktree->ReadFlag("timestampindex", fTimestampIndex); @@ -6643,17 +6647,22 @@ bool InitBlockIndex() { // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", true); pblocktree->WriteFlag("txindex", fTxIndex); + // Use the provided setting for -addressindex in the new database fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); - + + // Use the provided setting for -zindex in the new database + fAddressIndex = GetBoolArg("-zindex", DEFAULT_SHIELDEDINDEX); + pblocktree->WriteFlag("zindex", fZindex); + // Use the provided setting for -timestampindex in the new database fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); pblocktree->WriteFlag("timestampindex", fTimestampIndex); - + fSpentIndex = GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); pblocktree->WriteFlag("spentindex", fSpentIndex); - fprintf(stderr,"fAddressIndex.%d/%d fSpentIndex.%d/%d\n",fAddressIndex,DEFAULT_ADDRESSINDEX,fSpentIndex,DEFAULT_SPENTINDEX); + fprintf(stderr,"fAddressIndex.%d/%d fSpentIndex.%d/%d fZindex.%d/%d\n",fAddressIndex,DEFAULT_ADDRESSINDEX,fSpentIndex,DEFAULT_SPENTINDEX,fZindex, DEFAULT_SHIELDEDINDEX ); LogPrintf("Initializing databases...\n"); } // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 931954da0..08104a885 100644 --- a/src/main.h +++ b/src/main.h @@ -125,6 +125,7 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; #define DEFAULT_ADDRESSINDEX (GetArg("-ac_cc",0) != 0 || GetArg("-ac_ccactivate",0) != 0) #define DEFAULT_SPENTINDEX (GetArg("-ac_cc",0) != 0 || GetArg("-ac_ccactivate",0) != 0) static const bool DEFAULT_TIMESTAMPINDEX = false; +static const bool DEFAULT_SHIELDEDINDEX = false; static const unsigned int DEFAULT_DB_MAX_OPEN_FILES = 1000; static const bool DEFAULT_DB_COMPRESSION = true; From 44f34a22e75bc3d4c3fcbdc027b44c08f7e3d5fb Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Mon, 19 Aug 2019 05:07:34 -0700 Subject: [PATCH 17/22] more -zindex --- src/init.cpp | 1 + src/main.cpp | 112 ++++++++++++++++++++++++----------------- src/rpc/blockchain.cpp | 92 +++++++++++++++++---------------- 3 files changed, 118 insertions(+), 87 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 18985e842..9a1f0f666 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -400,6 +400,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); + strUsage += HelpMessageOpt("-zindex", strprintf(_("Maintain extra statistics about shielded transactions and payments (default: %u)"), 0)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), 100)); diff --git a/src/main.cpp b/src/main.cpp index f1057699f..41cc3a579 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4777,6 +4777,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl int64_t nShieldedSpends=0,nShieldedOutputs=0,nPayments=0; int64_t nShieldedTx=0,nFullyShieldedTx=0,nDeshieldingTx=0,nShieldingTx=0; int64_t nShieldedPayments=0,nFullyShieldedPayments=0,nShieldingPayments=0,nDeshieldingPayments=0; + int64_t nNotarizations=0; for (auto tx : block.vtx) { // Negative valueBalance "takes" money from the transparent value pool @@ -4789,10 +4790,19 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl sproutValue += js.vpub_old; sproutValue -= js.vpub_new; } + + if (!fZindex) + continue; + nShieldedSpends = tx.vShieldedSpend.size(); nShieldedOutputs = tx.vShieldedOutput.size(); isShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; + + if(tx.vin.size()==13 && tx.vout.size()==2) { + nNotarizations++; + } + if(isShieldedTx) { nShieldedTx++; if(tx.vin.size()==0 && tx.vout.size()==0) { @@ -4852,15 +4862,17 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - pindexNew->nPayments = nPayments; - pindexNew->nShieldedTx = nShieldedTx; - pindexNew->nFullyShieldedTx = nFullyShieldedTx; - pindexNew->nDeshieldingTx = nDeshieldingTx; - pindexNew->nShieldingTx = nShieldingTx; - pindexNew->nShieldedPayments = nShieldedPayments; - pindexNew->nFullyShieldedPayments = nFullyShieldedPayments; - pindexNew->nDeshieldingPayments = nDeshieldingPayments; - pindexNew->nShieldingPayments = nShieldingPayments; + if (fZindex) { + pindexNew->nPayments = nPayments; + pindexNew->nShieldedTx = nShieldedTx; + pindexNew->nFullyShieldedTx = nFullyShieldedTx; + pindexNew->nDeshieldingTx = nDeshieldingTx; + pindexNew->nShieldingTx = nShieldingTx; + pindexNew->nShieldedPayments = nShieldedPayments; + pindexNew->nFullyShieldedPayments = nFullyShieldedPayments; + pindexNew->nDeshieldingPayments = nDeshieldingPayments; + pindexNew->nShieldingPayments = nShieldingPayments; + } setDirtyBlockIndex.insert(pindexNew); @@ -4874,17 +4886,19 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; - pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; - pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; - pindex->nChainDeshieldingTx = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx; + if (fZindex) { + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; + pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; + pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; + pindex->nChainDeshieldingTx = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx; - pindex->nChainPayments = (pindex->pprev ? pindex->pprev->nChainPayments : 0) + pindex->nPayments; - pindex->nChainShieldedPayments = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments; - pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; - pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; - pindex->nChainDeshieldingPayments = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments; + pindex->nChainPayments = (pindex->pprev ? pindex->pprev->nChainPayments : 0) + pindex->nPayments; + pindex->nChainShieldedPayments = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments; + pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; + pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; + pindex->nChainDeshieldingPayments = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments; + } if (pindex->pprev) { if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { @@ -4922,8 +4936,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } } - fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, FullyShieldedTx=%d\n", - pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nFullyShieldedTx ); + if (fZindex) + fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, FullyShieldedTx=%d, ntz=%d\n", + pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nFullyShieldedTx, nNotarizations ); + return true; } @@ -6155,15 +6171,17 @@ bool static LoadBlockIndexDB() if (pindex->pprev) { if (pindex->pprev->nChainTx) { - pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; - pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; - pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; - pindex->nChainShieldingTx = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx; - pindex->nChainShieldingPayments = pindex->pprev->nChainShieldingPayments + pindex->nShieldingPayments; - pindex->nChainDeshieldingTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; - pindex->nChainDeshieldingPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; - pindex->nChainFullyShieldedTx = pindex->pprev->nChainFullyShieldedTx + pindex->nFullyShieldedTx; - pindex->nChainFullyShieldedPayments = pindex->pprev->nChainFullyShieldedPayments + pindex->nFullyShieldedPayments; + if (fZindex) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx; + pindex->nChainShieldingPayments = pindex->pprev->nChainShieldingPayments + pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainDeshieldingPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainFullyShieldedTx = pindex->pprev->nChainFullyShieldedTx + pindex->nFullyShieldedTx; + pindex->nChainFullyShieldedPayments = pindex->pprev->nChainFullyShieldedPayments + pindex->nFullyShieldedPayments; + } if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; @@ -6177,29 +6195,33 @@ bool static LoadBlockIndexDB() } } else { pindex->nChainTx = 0; - pindex->nChainShieldedTx = 0; - pindex->nChainFullyShieldedTx = 0; - pindex->nChainShieldedPayments = 0; - pindex->nChainShieldingPayments = 0; - pindex->nChainDeshieldingTx = 0; - pindex->nChainDeshieldingPayments = 0; - pindex->nChainFullyShieldedTx = 0; - pindex->nChainFullyShieldedPayments = 0; + if (fZindex) { + pindex->nChainShieldedTx = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainShieldedPayments = 0; + pindex->nChainShieldingPayments = 0; + pindex->nChainDeshieldingTx = 0; + pindex->nChainDeshieldingPayments = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainFullyShieldedPayments = 0; + } pindex->nChainSproutValue = boost::none; pindex->nChainSaplingValue = boost::none; mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); } } else { pindex->nChainTx = pindex->nTx; - pindex->nChainShieldedTx = pindex->nShieldedTx; - pindex->nChainShieldedPayments = pindex->nShieldedPayments; - pindex->nChainShieldingTx = pindex->nShieldingTx; - pindex->nChainShieldingPayments = pindex->nShieldingPayments; - pindex->nChainDeshieldingTx = pindex->nDeshieldingTx; - pindex->nChainDeshieldingPayments = pindex->nDeshieldingPayments; - pindex->nChainFullyShieldedPayments = pindex->nFullyShieldedPayments; pindex->nChainSproutValue = pindex->nSproutValue; pindex->nChainSaplingValue = pindex->nSaplingValue; + if (fZindex) { + pindex->nChainShieldedTx = pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->nShieldingTx; + pindex->nChainShieldingPayments = pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->nDeshieldingTx; + pindex->nChainDeshieldingPayments = pindex->nDeshieldingPayments; + pindex->nChainFullyShieldedPayments = pindex->nFullyShieldedPayments; + } } } // Construct in-memory chain of branch IDs. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d18f4e5c1..2df621066 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -49,6 +49,7 @@ using namespace std; extern int32_t KOMODO_INSYNC; +extern bool fZindex; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); int32_t komodo_notarized_height(int32_t *prevMoMheightp,uint256 *hashp,uint256 *txidp); @@ -2007,14 +2008,16 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("time", (int64_t)pindex->nTime); ret.pushKV("txcount", (int64_t)pindex->nChainTx); - ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); - ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nChainFullyShieldedTx); - ret.pushKV("deshielding_txcount", (int64_t)pindex->nChainDeshieldingTx); - ret.pushKV("shielding_txcount", (int64_t)pindex->nChainShieldingTx); - ret.pushKV("shielded_payments", (int64_t)pindex->nChainShieldedPayments); - ret.pushKV("fully_shielded_payments", (int64_t)pindex->nChainFullyShieldedPayments); - ret.pushKV("deshielding_payments", (int64_t)pindex->nChainDeshieldingPayments); - ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments); + if (fZindex) { + ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); + ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nChainFullyShieldedTx); + ret.pushKV("deshielding_txcount", (int64_t)pindex->nChainDeshieldingTx); + ret.pushKV("shielding_txcount", (int64_t)pindex->nChainShieldingTx); + ret.pushKV("shielded_payments", (int64_t)pindex->nChainShieldedPayments); + ret.pushKV("fully_shielded_payments", (int64_t)pindex->nChainFullyShieldedPayments); + ret.pushKV("deshielding_payments", (int64_t)pindex->nChainDeshieldingPayments); + ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments); + } ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); ret.pushKV("window_final_block_height", pindex->GetHeight()); @@ -2024,46 +2027,51 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("window_interval", nTimeDiff); ret.pushKV("window_txcount", nTxDiff); ret.pushKV("window_payments", nPaymentsDiff); - ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); - ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); - ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); - ret.pushKV("window_deshielding_payments", nDeshieldingPaymentsDiff); if (nTimeDiff > 0) { ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); - ret.pushKV("shielded_txrate", ((double)nShieldedTxDiff) / nTimeDiff); - ret.pushKV("fully_shielded_txrate", ((double)nFullyShieldedTxDiff) / nTimeDiff); - ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); - ret.pushKV("shielded_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); - ret.pushKV("fully_shielded_paymentrate", ((double)nFullyShieldedPaymentsDiff) / nTimeDiff); - } - if (nTxDiff > 0) { - ret.pushKV("shielded_tx_percent", ((double)pindex->nShieldedTx) / nTxDiff); - ret.pushKV("fully_shielded_tx_percent", ((double)pindex->nFullyShieldedTx) / nTxDiff); - ret.pushKV("shielding_tx_percent", ((double)pindex->nShieldingTx) / nTxDiff); - ret.pushKV("deshielding_tx_percent", ((double)pindex->nDeshieldingTx) / nTxDiff); - } - if (nPaymentsDiff > 0) { - ret.pushKV("shielded_payments_percent", ((double)pindex->nShieldedPayments) / nPaymentsDiff); - ret.pushKV("fully_shielded_payments_percent", ((double)pindex->nFullyShieldedPayments) / nPaymentsDiff); - ret.pushKV("shielding_payments_percent", ((double)pindex->nShieldingPayments) / nPaymentsDiff); - ret.pushKV("deshielding_payments_percent", ((double)pindex->nDeshieldingPayments) / nPaymentsDiff); + if (fZindex) { + ret.pushKV("shielded_txrate", ((double)nShieldedTxDiff) / nTimeDiff); + ret.pushKV("fully_shielded_txrate", ((double)nFullyShieldedTxDiff) / nTimeDiff); + ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); + ret.pushKV("shielded_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); + ret.pushKV("fully_shielded_paymentrate", ((double)nFullyShieldedPaymentsDiff) / nTimeDiff); + } } - // Shielded-only statistics - UniValue shielded(UniValue::VOBJ); - if (nShieldedTxDiff > 0) { - shielded.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nShieldedTxDiff ); - shielded.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nShieldedTxDiff ); - shielded.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nShieldedTxDiff ); + if (fZindex) { + ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); + ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); + ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); + ret.pushKV("window_deshielding_payments", nDeshieldingPaymentsDiff); + if (nTxDiff > 0) { + ret.pushKV("shielded_tx_percent", ((double)nShieldedTxDiff) / nTxDiff); + ret.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nTxDiff); + ret.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nTxDiff); + ret.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nTxDiff); + } + if (nPaymentsDiff > 0) { + ret.pushKV("shielded_payments_percent", ((double)nShieldedPaymentsDiff) / nPaymentsDiff); + ret.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nPaymentsDiff); + ret.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nPaymentsDiff); + ret.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nPaymentsDiff); + } + + // Shielded-only statistics + UniValue shielded(UniValue::VOBJ); + if (nShieldedTxDiff > 0) { + shielded.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nShieldedTxDiff ); + shielded.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nShieldedTxDiff ); + shielded.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nShieldedTxDiff ); + } + if (nShieldedPaymentsDiff > 0) { + shielded.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nShieldedPaymentsDiff ); + } + if(nShieldedTxDiff+nShieldedPaymentsDiff > 0) + ret.pushKV("shielded_only", shielded); } - if (nShieldedPaymentsDiff > 0) { - shielded.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nShieldedPaymentsDiff ); - shielded.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nShieldedPaymentsDiff ); - shielded.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nShieldedPaymentsDiff ); - } - if(nShieldedTxDiff+nShieldedPaymentsDiff > 0) - ret.pushKV("shielded_only", shielded); } return ret; From b2c03c82a57b1a521f47cd63df53e4d043624fb9 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Mon, 19 Aug 2019 05:25:25 -0700 Subject: [PATCH 18/22] Explain about changing heuristics + on-disk data --- src/main.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 41cc3a579..8ed78f512 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4791,6 +4791,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl sproutValue -= js.vpub_new; } + // Ignore following stats unless -zindex if (!fZindex) continue; @@ -4812,8 +4813,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } else if(tx.vout.size()>0) { nDeshieldingTx++; } - //NOTE: These are at best heuristics. Improve them as much as possible - // You cannot compare stats generated from different sets of heuristics + //NOTE: These are at best heuristics. Improve them as much as possible. + // You cannot compare stats generated from different sets of heuristics, so + // if you change this code, you must delete and resync from scratch, or you + // will be mixing together data from two set of heuristics. if (nShieldedOutputs >= 1) { // If there are shielded outputs, count each as a payment @@ -6299,9 +6302,11 @@ bool static LoadBlockIndexDB() // Check whether we have a transaction index pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); + // Check whether we have an address index pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a shielded index pblocktree->ReadFlag("zindex", fZindex); LogPrintf("%s: shielded index %s\n", __func__, fZindex ? "enabled" : "disabled"); From d03d774c184d400f88288abe732a712b12700496 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Tue, 20 Aug 2019 20:19:24 -0700 Subject: [PATCH 19/22] Fix zindex flag detection --- src/init.cpp | 8 +++----- src/main.cpp | 2 +- src/main.h | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 9a1f0f666..8d37f2a46 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1283,14 +1283,11 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) return InitError(strprintf(_("Cannot obtain a lock on data directory %s. Komodo is probably already running.") + " %s.", strDataDir, e.what())); } - fprintf(stderr,"About to create pidfile\n"); #ifndef _WIN32 CreatePidFile(GetPidFile(), getpid()); #endif - fprintf(stderr,"created pidfile\n"); if (GetBoolArg("-shrinkdebugfile", !fDebug)) ShrinkDebugFile(); - fprintf(stderr,"past shrinkdebugfile\n"); LogPrintf("\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"); LogPrintf("Komodo version %s (%s)\n", FormatFullVersion(), CLIENT_DATE); @@ -1669,8 +1666,9 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) break; } - // Check for changed -zindex state - if (fZindex != GetBoolArg("-zindex", false)) { + fprintf(stderr, "zindex=%s in block index??\n", fZindex ? "enabled" : "disabled"); + // Turning on -zindex requires a reindex, turning it off doesn't + if (fZindex && (fZindex != GetBoolArg("-zindex", false))) { strLoadError = _("You need to rebuild the database using -reindex to change -zindex"); break; } diff --git a/src/main.cpp b/src/main.cpp index 8ed78f512..524110948 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6680,7 +6680,7 @@ bool InitBlockIndex() { pblocktree->WriteFlag("addressindex", fAddressIndex); // Use the provided setting for -zindex in the new database - fAddressIndex = GetBoolArg("-zindex", DEFAULT_SHIELDEDINDEX); + fZindex = GetBoolArg("-zindex", DEFAULT_SHIELDEDINDEX); pblocktree->WriteFlag("zindex", fZindex); // Use the provided setting for -timestampindex in the new database diff --git a/src/main.h b/src/main.h index 08104a885..800f05745 100644 --- a/src/main.h +++ b/src/main.h @@ -158,6 +158,7 @@ extern bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; +extern bool fZindex; extern bool fIsBareMultisigStd; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; From 3b49c1d256e198ba7ba036ec1b892054f90ce514 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 21 Aug 2019 06:43:41 -0700 Subject: [PATCH 20/22] Improve ntz detection and serialize to disk --- src/chain.h | 12 ++++++++++-- src/main.cpp | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/chain.h b/src/chain.h index 1a0a7057d..0ae37a90e 100644 --- a/src/chain.h +++ b/src/chain.h @@ -260,6 +260,10 @@ public: //! Note: in a potential headers-first mode, this number cannot be relied upon unsigned int nTx; + //! Number of notarization transactions in this block. + unsigned int nNotarizations; + + // TODO: convert block-stats to unsigned int ? //! (memory only) Number of payments (shielded or transparent) in the block //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent @@ -310,6 +314,9 @@ public: //! Change to 64-bit type when necessary; won't happen before 2030 unsigned int nChainTx; + //! Number of notarization transactions in this chain + int64_t nChainNotarizations; + //! (memory only) Number of payments (shielded or transparent) in the chain //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent @@ -615,8 +622,7 @@ public: READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); - // These values only serialized when -zindex=1 - //if (fZindex != GetBoolArg("-zindex", false)) { + // These values only serialized when -zindex enabled if(fZindex) { READWRITE(VARINT(nShieldedTx)); READWRITE(VARINT(nShieldingTx)); @@ -627,6 +633,8 @@ public: READWRITE(VARINT(nShieldingPayments)); READWRITE(VARINT(nDeshieldingPayments)); READWRITE(VARINT(nFullyShieldedPayments)); + + READWRITE(VARINT(nChainNotarizations)); } if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) diff --git a/src/main.cpp b/src/main.cpp index 524110948..b970663f7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4799,8 +4799,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedOutputs = tx.vShieldedOutput.size(); isShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; - - if(tx.vin.size()==13 && tx.vout.size()==2) { + // We want to avoid full verification with a low false-positive rate + if(tx.vin.size()==13 && tx.vout.size()==2 && tx.vout[1].scriptPubKey.IsOpReturn() && tx.vout[1].nValue==0) { nNotarizations++; } @@ -4875,6 +4875,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nFullyShieldedPayments = nFullyShieldedPayments; pindexNew->nDeshieldingPayments = nDeshieldingPayments; pindexNew->nShieldingPayments = nShieldingPayments; + pindexNew->nNotarizations = nNotarizations; } setDirtyBlockIndex.insert(pindexNew); @@ -4891,6 +4892,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl if (fZindex) { pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + pindex->nChainNotarizations = (pindex->pprev ? pindex->pprev->nChainNotarizations : 0) + pindex->nNotarizations; pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; @@ -6175,6 +6177,7 @@ bool static LoadBlockIndexDB() if (pindex->pprev->nChainTx) { if (fZindex) { + pindex->nChainNotarizations = pindex->pprev->nChainNotarizations + pindex->nNotarizations; pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; @@ -6199,6 +6202,7 @@ bool static LoadBlockIndexDB() } else { pindex->nChainTx = 0; if (fZindex) { + pindex->nChainNotarizations = 0; pindex->nChainShieldedTx = 0; pindex->nChainFullyShieldedTx = 0; pindex->nChainShieldedPayments = 0; @@ -6217,6 +6221,7 @@ bool static LoadBlockIndexDB() pindex->nChainSproutValue = pindex->nSproutValue; pindex->nChainSaplingValue = pindex->nSaplingValue; if (fZindex) { + pindex->nChainNotarizations = pindex->nNotarizations; pindex->nChainShieldedTx = pindex->nShieldedTx; pindex->nChainShieldedPayments = pindex->nShieldedPayments; pindex->nChainShieldingTx = pindex->nShieldingTx; From c91d1d1fdfabae2c496a0dabe1344698cdfe2137 Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 21 Aug 2019 11:25:38 -0700 Subject: [PATCH 21/22] Use uints to save RAM/disk on block stats; fix ntz serialization --- src/chain.h | 23 ++++++++++------------- src/rpc/blockchain.cpp | 25 +++++++++++++++---------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/chain.h b/src/chain.h index 0ae37a90e..f6c2bf980 100644 --- a/src/chain.h +++ b/src/chain.h @@ -263,50 +263,47 @@ public: //! Number of notarization transactions in this block. unsigned int nNotarizations; - // TODO: convert block-stats to unsigned int ? //! (memory only) Number of payments (shielded or transparent) in the block //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent //! when combined with shielded payment stats - int64_t nPayments; + unsigned int nPayments; //! (memory only) Number of shielded transactions (of any kind) in the block up to and including this block. //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. - int64_t nShieldedTx; + unsigned int nShieldedTx; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... - int64_t nFullyShieldedTx; + unsigned int nFullyShieldedTx; //! (memory only) Number of shielding payments. A shielding payment is defined //! as having a shielded output but transparent input: t->z - int64_t nShieldingPayments; + unsigned int nShieldingPayments; //! (memory only) Number of shielded payments. A shielded payment is defined //! as having a shielded input or output: t->z or z->t - int64_t nShieldedPayments; + unsigned int nShieldedPayments; //! (memory only) Number of fully shielded payments. A fully shielded payment is defined //! as having a shielded input and shielded output: z->z - int64_t nFullyShieldedPayments; + unsigned int nFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent output. - int64_t nDeshieldingTx; + unsigned int nDeshieldingTx; //! (memory only) Number of deshielding payments. A deshielding payment is defined //! as one transparent input and one shielded output: z->t - int64_t nDeshieldingPayments; + unsigned int nDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent input // i.e. t->z or t->(z,t) or z->(z,z,t) - int64_t nShieldingTx; - - + unsigned int nShieldingTx; // chain stats //! (memory only) Number of transactions in the chain up to and including this block. @@ -634,7 +631,7 @@ public: READWRITE(VARINT(nDeshieldingPayments)); READWRITE(VARINT(nFullyShieldedPayments)); - READWRITE(VARINT(nChainNotarizations)); + READWRITE(VARINT(nNotarizations)); } if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 2df621066..f965fdeef 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1994,21 +1994,13 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; - int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; - int64_t nShieldedTxDiff = pindex->nChainShieldedTx - pindexPast->nChainShieldedTx; - int64_t nShieldingTxDiff = pindex->nChainShieldingTx - pindexPast->nChainShieldingTx; - int64_t nDeshieldingTxDiff = pindex->nChainDeshieldingTx - pindexPast->nChainDeshieldingTx; - int64_t nFullyShieldedTxDiff = pindex->nChainFullyShieldedTx - pindexPast->nChainFullyShieldedTx; - int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; - int64_t nShieldingPaymentsDiff = pindex->nChainShieldingPayments - pindexPast->nChainShieldingPayments; - int64_t nDeshieldingPaymentsDiff = pindex->nChainDeshieldingPayments - pindexPast->nChainDeshieldingPayments; - int64_t nFullyShieldedPaymentsDiff = pindex->nChainFullyShieldedPayments - pindexPast->nChainFullyShieldedPayments; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); - ret.pushKV("txcount", (int64_t)pindex->nChainTx); + if (fZindex) { + ret.pushKV("notarizations", (int64_t)pindex->nChainNotarizations); ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nChainFullyShieldedTx); ret.pushKV("deshielding_txcount", (int64_t)pindex->nChainDeshieldingTx); @@ -2024,13 +2016,26 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp) ret.pushKV("window_block_count", blockcount); if (blockcount > 0) { + int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; + int64_t nShieldedTxDiff = pindex->nChainShieldedTx - pindexPast->nChainShieldedTx; + int64_t nShieldingTxDiff = pindex->nChainShieldingTx - pindexPast->nChainShieldingTx; + int64_t nDeshieldingTxDiff = pindex->nChainDeshieldingTx - pindexPast->nChainDeshieldingTx; + int64_t nFullyShieldedTxDiff = pindex->nChainFullyShieldedTx - pindexPast->nChainFullyShieldedTx; + int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; + int64_t nShieldingPaymentsDiff = pindex->nChainShieldingPayments - pindexPast->nChainShieldingPayments; + int64_t nDeshieldingPaymentsDiff = pindex->nChainDeshieldingPayments - pindexPast->nChainDeshieldingPayments; + int64_t nFullyShieldedPaymentsDiff = pindex->nChainFullyShieldedPayments - pindexPast->nChainFullyShieldedPayments; + unsigned int nNotarizationsDiff = pindex->nChainNotarizations - pindexPast->nChainNotarizations; + ret.pushKV("window_interval", nTimeDiff); ret.pushKV("window_txcount", nTxDiff); ret.pushKV("window_payments", nPaymentsDiff); + ret.pushKV("window_notarizations", (int) nNotarizationsDiff); if (nTimeDiff > 0) { ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); if (fZindex) { + ret.pushKV("notarization_txrate", ((double)nNotarizationsDiff) / nTimeDiff); ret.pushKV("shielded_txrate", ((double)nShieldedTxDiff) / nTimeDiff); ret.pushKV("fully_shielded_txrate", ((double)nFullyShieldedTxDiff) / nTimeDiff); ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); From 76543056abb4c2d03d02b7dfe459db56cea70edb Mon Sep 17 00:00:00 2001 From: "Jonathan \"Duke\" Leto" Date: Wed, 21 Aug 2019 19:50:50 -0700 Subject: [PATCH 22/22] Changing -zindex on or off requires reindex --- src/init.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 8d37f2a46..a12cad1dd 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1666,9 +1666,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) break; } - fprintf(stderr, "zindex=%s in block index??\n", fZindex ? "enabled" : "disabled"); - // Turning on -zindex requires a reindex, turning it off doesn't - if (fZindex && (fZindex != GetBoolArg("-zindex", false))) { + fprintf(stderr, "zindex=%s in block index\n", fZindex ? "enabled" : "disabled"); + if (fZindex != GetBoolArg("-zindex", false)) { strLoadError = _("You need to rebuild the database using -reindex to change -zindex"); break; }