From 2492ac8c3a8d0a2b9c4812136a3e003fdde863ef Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Tue, 17 Dec 2019 09:43:29 -0500 Subject: [PATCH 01/21] Start scavenging zindex code from the depths of the git bit bucket --- src/chain.h | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/chain.h b/src/chain.h index c58127c42..f564a9a3c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -256,17 +256,61 @@ public: //! (memory only) Total amount of work (expected number of hashes) in the chain up to and including this block CChainPower chainPower; - // block stats //! Number of transactions in this block. //! Note: in a potential headers-first mode, this number cannot be relied upon unsigned int nTx; - // chain stats //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. //! Change to 64-bit type when necessary; won't happen before 2030 unsigned int nChainTx; + //! (memory only) Number of payments (shielded or transparent) in the chain + //! up to and including this block. One transaction can contain one or more + //! payments. This stat allows us to calculate ratios of shielded/transparent + //! when combined with shielded payment stats + int64_t nChainPayments; + + //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. + //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as + //! inputs and outputs. + int64_t nShieldedTx; + + //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined + //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... + int64_t nFullyShieldedTx; + + //! (memory only) Number of shielding payments. A shielding payment is defined + //! as having a shielded output but transparent input: t->z + int64_t nShieldingPayments; + + //! (memory only) Number of shielded payments. A shielded payment is defined + //! as having a shielded input or output: t->z or z->t + int64_t nShieldedPayments; + + //! (memory only) Number of fully shielded payments. A fully shielded payment is defined + //! as having a shielded input and shielded output: z->z + int64_t nFullyShieldedPayments; + + //! (memory only) Number of deshielding transactions. A deshielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent output. + int64_t nDeshieldingTx; + + //! (memory only) Number of deshielding payments. A deshielding payment is defined + //! as one transparent input and one shielded output: z->t + int64_t nDeshieldingPayments; + + //! (memory only) Number of shielding transactions. A shielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent input + // i.e. t->z or t->(z,t) or z->(z,z,t) + int64_t nShieldingTx; + + //! (memory only) Number of shielded transactions which contain multiple shielded input notes + //! z_sendmany cannot make these and so it indicates raw shielded transaction usage currently + int64_t nMultipleShieldedInputsTx; + //! Verification status of this block. See enum BlockStatus unsigned int nStatus; @@ -325,7 +369,15 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; - + nShieldedTx = 0; + nShieldingTx = 0; + nDeshieldingTx = 0; + nFullyShieldedTx = 0; + nShieldedPayments = 0; + nShieldingPayments = 0; + nDeshieldingPayments = 0; + nFullyShieldedPayments = 0; + nMultipleShieldedInputsTx = 0; nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); From fbfdc8f40fba01826c544b129b6c2d55c3f9a438 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Tue, 17 Dec 2019 12:08:14 -0500 Subject: [PATCH 02/21] Port zindex code from old patches; compiles and seems to work when zindex is disabled --- src/chain.h | 95 +++++++++++++++++++++++++---- src/main.cpp | 133 +++++++++++++++++++++++++++++++++++++++++ src/rpc/blockchain.cpp | 76 ++++++++++++++++++++++- src/txdb.cpp | 15 ++++- 4 files changed, 301 insertions(+), 18 deletions(-) diff --git a/src/chain.h b/src/chain.h index f564a9a3c..b596ad75a 100644 --- a/src/chain.h +++ b/src/chain.h @@ -138,6 +138,7 @@ class CBlockIndex; // that involves mining in secret completely ineffective, even before dPOW, unless a large part // of the staking supply is also controlled. It also enables a faster deterministic convergence, // aided by both POS and POW. +// TODO: delete this junk class CChainPower { public: @@ -260,11 +261,59 @@ public: //! Note: in a potential headers-first mode, this number cannot be relied upon unsigned int nTx; + //! Number of notarization transactions in this block. + unsigned int nNotarizations; + + //! (memory only) Number of payments (shielded or transparent) in the block + //! up to and including this block. One transaction can contain one or more + //! payments. This stat allows us to calculate ratios of shielded/transparent + //! when combined with shielded payment stats + unsigned int nPayments; + + //! (memory only) Number of shielded transactions (of any kind) in the block up to and including this block. + //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as + //! inputs and outputs. + unsigned int nShieldedTx; + + //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined + //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... + unsigned int nFullyShieldedTx; + + //! (memory only) Number of shielding payments. A shielding payment is defined + //! as having a shielded output but transparent input: t->z + unsigned int nShieldingPayments; + + //! (memory only) Number of shielded payments. A shielded payment is defined + //! as having a shielded input or output: t->z or z->t + unsigned int nShieldedPayments; + + //! (memory only) Number of fully shielded payments. A fully shielded payment is defined + //! as having a shielded input and shielded output: z->z + unsigned int nFullyShieldedPayments; + + //! (memory only) Number of deshielding transactions. A deshielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent output. + unsigned int nDeshieldingTx; + + //! (memory only) Number of deshielding payments. A deshielding payment is defined + //! as one transparent input and one shielded output: z->t + unsigned int nDeshieldingPayments; + + //! (memory only) Number of shielding transactions. A shielding transaction is defined + //! as a transaction containing JoinSplits and at least one transparent input + // i.e. t->z or t->(z,t) or z->(z,z,t) + unsigned int nShieldingTx; + //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. //! Change to 64-bit type when necessary; won't happen before 2030 unsigned int nChainTx; + //! Number of notarization transactions in this chain + int64_t nChainNotarizations; + //! (memory only) Number of payments (shielded or transparent) in the chain //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent @@ -275,41 +324,37 @@ public: //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. - int64_t nShieldedTx; + int64_t nChainShieldedTx; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... - int64_t nFullyShieldedTx; + int64_t nChainFullyShieldedTx; //! (memory only) Number of shielding payments. A shielding payment is defined //! as having a shielded output but transparent input: t->z - int64_t nShieldingPayments; + int64_t nChainShieldingPayments; //! (memory only) Number of shielded payments. A shielded payment is defined //! as having a shielded input or output: t->z or z->t - int64_t nShieldedPayments; + int64_t nChainShieldedPayments; //! (memory only) Number of fully shielded payments. A fully shielded payment is defined //! as having a shielded input and shielded output: z->z - int64_t nFullyShieldedPayments; + int64_t nChainFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent output. - int64_t nDeshieldingTx; + int64_t nChainDeshieldingTx; //! (memory only) Number of deshielding payments. A deshielding payment is defined //! as one transparent input and one shielded output: z->t - int64_t nDeshieldingPayments; + int64_t nChainDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent input // i.e. t->z or t->(z,t) or z->(z,z,t) - int64_t nShieldingTx; - - //! (memory only) Number of shielded transactions which contain multiple shielded input notes - //! z_sendmany cannot make these and so it indicates raw shielded transaction usage currently - int64_t nMultipleShieldedInputsTx; + int64_t nChainShieldingTx; //! Verification status of this block. See enum BlockStatus unsigned int nStatus; @@ -369,6 +414,16 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; + nChainPayments = 0; + nChainShieldedTx = 0; + nChainShieldingTx = 0; + nChainDeshieldingTx = 0; + nChainNotarizations = 0; + nChainFullyShieldedTx = 0; + nChainShieldedPayments = 0; + nChainShieldingPayments = 0; + nChainDeshieldingPayments = 0; + nChainFullyShieldedPayments = 0; nShieldedTx = 0; nShieldingTx = 0; nDeshieldingTx = 0; @@ -377,7 +432,6 @@ public: nShieldingPayments = 0; nDeshieldingPayments = 0; nFullyShieldedPayments = 0; - nMultipleShieldedInputsTx = 0; nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); @@ -553,6 +607,21 @@ public: READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); + // These values only serialized when -zindex enabled + if((s.GetType() & SER_DISK) && fZindex) { + READWRITE(VARINT(nShieldedTx)); + READWRITE(VARINT(nShieldingTx)); + READWRITE(VARINT(nDeshieldingTx)); + READWRITE(VARINT(nFullyShieldedTx)); + + READWRITE(VARINT(nShieldedPayments)); + READWRITE(VARINT(nShieldingPayments)); + READWRITE(VARINT(nDeshieldingPayments)); + READWRITE(VARINT(nFullyShieldedPayments)); + + READWRITE(VARINT(nNotarizations)); + } + if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(nFile)); if (nStatus & BLOCK_HAVE_DATA) diff --git a/src/main.cpp b/src/main.cpp index a695a3a64..b137a04c9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4717,6 +4717,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CAmount sproutValue = 0; CAmount saplingValue = 0; bool isShieldedTx = false; + unsigned int nShieldedSpends=0,nShieldedOutputs=0,nPayments=0; + unsigned int nShieldedTx=0,nFullyShieldedTx=0,nDeshieldingTx=0,nShieldingTx=0; + unsigned int nShieldedPayments=0,nFullyShieldedPayments=0,nShieldingPayments=0,nDeshieldingPayments=0; + unsigned int nNotarizations=0; for (auto tx : block.vtx) { // Negative valueBalance "takes" money from the transparent value pool @@ -4729,6 +4733,69 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl sproutValue += js.vpub_old; sproutValue -= js.vpub_new; } + + // Ignore following stats unless -zindex enabled + if (!fZindex) + continue; + + nShieldedSpends = tx.vShieldedSpend.size(); + nShieldedOutputs = tx.vShieldedOutput.size(); + isShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; + + // We want to avoid full verification with a low false-positive rate + if(tx.vin.size()==13 && tx.vout.size()==2 && tx.vout[1].scriptPubKey.IsOpReturn() && tx.vout[1].nValue==0) { + nNotarizations++; + } + + if(isShieldedTx) { + nShieldedTx++; + if(tx.vin.size()==0 && tx.vout.size()==0) { + nFullyShieldedTx++; + } else if(tx.vin.size()>0) { + nShieldingTx++; + } else if(tx.vout.size()>0) { + nDeshieldingTx++; + } + //NOTE: These are at best heuristics. Improve them as much as possible. + // You cannot compare stats generated from different sets of heuristics, so + // if you change this code, you must delete and resync from scratch, or you + // will be mixing together data from two set of heuristics. + + if (nShieldedOutputs >= 1) { + // If there are shielded outputs, count each as a payment + // By default, if there is more than 1 output, we assume 1 change output which is not a payment. + // In the case of multiple outputs which spend inputs exactly, there is no change output and this + // heuristic will undercount payments. Since this edge case is rare, this seems acceptable. + // t->(t,t,z) = 1 shielded payment + // z->(z,z) = 1 shielded payment + shielded change + // t->(z,z) = 1 shielded payment + shielded change + // t->(t,z) = 1 shielded payment + transparent change + // (z,z)->z = 1 shielded payment (has this xtn ever occurred?) + // z->(z,z,z) = 2 shielded payments + shielded change + // Assume that there is always 1 change output when there are more than one + nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + + // Fully shielded do not count toward shielding/deshielding + if(tx.vin.size()==0 && tx.vout.size()==0) { + nFullyShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + } else { + nShieldingPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + } + } else if (nShieldedSpends >=1) { + // Shielded inputs with no shielded outputs. We know none are change output because + // change would flow back to the zaddr + // z->t = 1 shielded payment + // z->(t,t) = 2 shielded payments + // z->(t,t,t) = 3 shielded payments + nShieldedPayments += tx.vout.size(); + nDeshieldingPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); + } + //TODO: correctly add shielded payments to total chain payments + nPayments += nShieldedPayments; + } else { + // No shielded payments, add transparent payments minus a change address + nPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); + } } pindexNew->nSproutValue = sproutValue; pindexNew->nChainSproutValue = boost::none; @@ -4739,6 +4806,19 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); + + if (fZindex) { + pindexNew->nPayments = nPayments; + pindexNew->nShieldedTx = nShieldedTx; + pindexNew->nFullyShieldedTx = nFullyShieldedTx; + pindexNew->nDeshieldingTx = nDeshieldingTx; + pindexNew->nShieldingTx = nShieldingTx; + pindexNew->nShieldedPayments = nShieldedPayments; + pindexNew->nFullyShieldedPayments = nFullyShieldedPayments; + pindexNew->nDeshieldingPayments = nDeshieldingPayments; + pindexNew->nShieldingPayments = nShieldingPayments; + pindexNew->nNotarizations = nNotarizations; + } setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { @@ -4751,6 +4831,22 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + + if (fZindex) { + fprintf(stderr,"%s: setting chain zstats\n", __FUNCTION__); + pindex->nChainNotarizations = (pindex->pprev ? pindex->pprev->nChainNotarizations : 0) + pindex->nNotarizations; + pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; + pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; + pindex->nChainShieldingTx = (pindex->pprev ? pindex->pprev->nChainShieldingTx : 0) + pindex->nShieldingTx; + pindex->nChainDeshieldingTx = (pindex->pprev ? pindex->pprev->nChainDeshieldingTx : 0) + pindex->nDeshieldingTx; + + pindex->nChainPayments = (pindex->pprev ? pindex->pprev->nChainPayments : 0) + pindex->nPayments; + pindex->nChainShieldedPayments = (pindex->pprev ? pindex->pprev->nChainShieldedPayments : 0) + pindex->nShieldedPayments; + pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; + pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; + pindex->nChainDeshieldingPayments = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments; + } + if (pindex->pprev) { if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; @@ -4787,6 +4883,9 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } } + if (fZindex) + fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, FullyShieldedTx=%d, ntz=%d\n", pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nFullyShieldedTx, nNotarizations ); + return true; } @@ -6035,6 +6134,19 @@ bool static LoadBlockIndexDB() if (pindex->pprev) { if (pindex->pprev->nChainTx) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + if (fZindex) { + pindex->nChainNotarizations = pindex->pprev->nChainNotarizations + pindex->nNotarizations; + pindex->nChainShieldedTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx; + + pindex->nChainShieldingPayments = pindex->pprev->nChainShieldingPayments + pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; + pindex->nChainDeshieldingPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; + pindex->nChainFullyShieldedTx = pindex->pprev->nChainFullyShieldedTx + pindex->nFullyShieldedTx; + pindex->nChainFullyShieldedPayments = pindex->pprev->nChainFullyShieldedPayments + pindex->nFullyShieldedPayments; + } + if (pindex->pprev->nChainSproutValue && pindex->nSproutValue) { pindex->nChainSproutValue = *pindex->pprev->nChainSproutValue + *pindex->nSproutValue; } else { @@ -6047,6 +6159,17 @@ bool static LoadBlockIndexDB() } } else { pindex->nChainTx = 0; + if (fZindex) { + pindex->nChainNotarizations = 0; + pindex->nChainShieldedTx = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainShieldedPayments = 0; + pindex->nChainShieldingPayments = 0; + pindex->nChainDeshieldingTx = 0; + pindex->nChainDeshieldingPayments = 0; + pindex->nChainFullyShieldedTx = 0; + pindex->nChainFullyShieldedPayments = 0; + } pindex->nChainSproutValue = boost::none; pindex->nChainSaplingValue = boost::none; mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); @@ -6055,6 +6178,16 @@ bool static LoadBlockIndexDB() pindex->nChainTx = pindex->nTx; pindex->nChainSproutValue = pindex->nSproutValue; pindex->nChainSaplingValue = pindex->nSaplingValue; + if (fZindex) { + pindex->nChainNotarizations = pindex->nNotarizations; + pindex->nChainShieldedTx = pindex->nShieldedTx; + pindex->nChainShieldedPayments = pindex->nShieldedPayments; + pindex->nChainShieldingTx = pindex->nShieldingTx; + pindex->nChainShieldingPayments = pindex->nShieldingPayments; + pindex->nChainDeshieldingTx = pindex->nDeshieldingTx; + pindex->nChainDeshieldingPayments = pindex->nDeshieldingPayments; + pindex->nChainFullyShieldedPayments = pindex->nFullyShieldedPayments; + } } } // Construct in-memory chain of branch IDs. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8b2976bf6..0ff5a1b40 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1929,20 +1929,90 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk } const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); - int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); - int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); ret.pushKV("txcount", (int64_t)pindex->nChainTx); ret.pushKV("window_final_block_hash", pindex->GetBlockHash().GetHex()); + ret.pushKV("window_final_block_height", pindex->GetHeight()); ret.pushKV("window_block_count", blockcount); + + if (fZindex) { + ret.pushKV("notarizations", (int64_t)pindex->nChainNotarizations); + ret.pushKV("shielded_txcount", (int64_t)pindex->nChainShieldedTx); + ret.pushKV("fully_shielded_txcount", (int64_t)pindex->nChainFullyShieldedTx); + ret.pushKV("deshielding_txcount", (int64_t)pindex->nChainDeshieldingTx); + ret.pushKV("shielding_txcount", (int64_t)pindex->nChainShieldingTx); + ret.pushKV("shielded_payments", (int64_t)pindex->nChainShieldedPayments); + ret.pushKV("fully_shielded_payments", (int64_t)pindex->nChainFullyShieldedPayments); + ret.pushKV("deshielding_payments", (int64_t)pindex->nChainDeshieldingPayments); + ret.pushKV("shielding_payments", (int64_t)pindex->nChainShieldingPayments); + } + if (blockcount > 0) { ret.pushKV("window_tx_count", nTxDiff); ret.pushKV("window_interval", nTimeDiff); + int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; + int64_t nShieldedTxDiff = pindex->nChainShieldedTx - pindexPast->nChainShieldedTx; + int64_t nShieldingTxDiff = pindex->nChainShieldingTx - pindexPast->nChainShieldingTx; + int64_t nDeshieldingTxDiff = pindex->nChainDeshieldingTx - pindexPast->nChainDeshieldingTx; + int64_t nFullyShieldedTxDiff = pindex->nChainFullyShieldedTx - pindexPast->nChainFullyShieldedTx; + int64_t nShieldedPaymentsDiff = pindex->nChainShieldedPayments - pindexPast->nChainShieldedPayments; + int64_t nShieldingPaymentsDiff = pindex->nChainShieldingPayments - pindexPast->nChainShieldingPayments; + int64_t nDeshieldingPaymentsDiff = pindex->nChainDeshieldingPayments - pindexPast->nChainDeshieldingPayments; + int64_t nFullyShieldedPaymentsDiff = pindex->nChainFullyShieldedPayments - pindexPast->nChainFullyShieldedPayments; + int64_t nNotarizationsDiff = pindex->nChainNotarizations - pindexPast->nChainNotarizations; + + ret.pushKV("window_payments", (int) nPaymentsDiff); + ret.pushKV("window_notarizations", (int) nNotarizationsDiff); + if (nTimeDiff > 0) { - ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); + if (fZindex) { + ret.pushKV("notarization_txrate", ((double)nNotarizationsDiff) / nTimeDiff); + ret.pushKV("shielded_txrate", ((double)nShieldedTxDiff) / nTimeDiff); + ret.pushKV("fully_shielded_txrate", ((double)nFullyShieldedTxDiff) / nTimeDiff); + ret.pushKV("paymentrate", ((double)nPaymentsDiff) / nTimeDiff); + ret.pushKV("shielded_paymentrate", ((double)nShieldedPaymentsDiff) / nTimeDiff); + ret.pushKV("fully_shielded_paymentrate", ((double)nFullyShieldedPaymentsDiff) / nTimeDiff); + } } + + if (fZindex) { + ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); + ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); + ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); + ret.pushKV("window_deshielding_payments", nDeshieldingPaymentsDiff); + if (nTxDiff > 0) { + ret.pushKV("shielded_tx_percent", ((double)nShieldedTxDiff) / nTxDiff); + ret.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nTxDiff); + ret.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nTxDiff); + ret.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nTxDiff); + } + if (nPaymentsDiff > 0) { + ret.pushKV("shielded_payments_percent", ((double)nShieldedPaymentsDiff) / nPaymentsDiff); + ret.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nPaymentsDiff); + ret.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nPaymentsDiff); + ret.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nPaymentsDiff); + } + + // Shielded statistics + UniValue shielded(UniValue::VOBJ); + if (nShieldedTxDiff > 0) { + shielded.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nShieldedTxDiff ); + shielded.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / nShieldedTxDiff ); + shielded.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / nShieldedTxDiff ); + } + if (nShieldedPaymentsDiff > 0) { + shielded.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nShieldedPaymentsDiff ); + shielded.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nShieldedPaymentsDiff ); + } + if(nShieldedTxDiff+nShieldedPaymentsDiff > 0) + ret.pushKV("shielded", shielded); + } } return ret; diff --git a/src/txdb.cpp b/src/txdb.cpp index a2ae3fb21..334977f2d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,7 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2019 The Hush developers + // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,7 +35,7 @@ using namespace std; -// NOTE: Per issue #3277, do not use the prefix 'X' or 'x' as they were +// NOTE: Per zcash issue #3277, do not use the prefix 'X' or 'x' as they were // previously used by DB_SAPLING_ANCHOR and DB_BEST_SAPLING_ANCHOR. static const char DB_SPROUT_ANCHOR = 'A'; static const char DB_SAPLING_ANCHOR = 'Z'; @@ -720,8 +722,17 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nSaplingValue = diskindex.nSaplingValue; pindexNew->segid = diskindex.segid; pindexNew->nNotaryPay = diskindex.nNotaryPay; + pindexNew->nShieldedTx = diskindex.nShieldedTx; + pindexNew->nShieldedPayments = diskindex.nShieldedPayments; + pindexNew->nShieldingTx = diskindex.nShieldingTx; + pindexNew->nShieldingPayments = diskindex.nShieldingPayments; + pindexNew->nDeshieldingTx = diskindex.nDeshieldingTx; + pindexNew->nDeshieldingPayments = diskindex.nDeshieldingPayments; + pindexNew->nFullyShieldedTx = diskindex.nFullyShieldedTx; + pindexNew->nFullyShieldedPayments = diskindex.nFullyShieldedPayments; + pindexNew->nNotarizations = diskindex.nNotarizations; -//fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); + fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks auto header = pindexNew->GetBlockHeader(); if (header.GetHash() != pindexNew->GetBlockHash()) From 1cbcb0d012f7e60ca46585cf084a3b6522fa9444 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Tue, 17 Dec 2019 18:23:33 -0500 Subject: [PATCH 03/21] Organic stats --- src/rpc/blockchain.cpp | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 0ff5a1b40..d4fe63f8b 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1874,12 +1874,17 @@ inline CBlockIndex* LookupBlockIndex(const uint256& hash) return it == mapBlockIndex.end() ? nullptr : it->second; } +// given a transaction count X, subtract out coinbase and dpow transactions +// to give an "organic count" +#define ORG(X) (X - blockcount - nNotarizationsDiff) + UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk) { if (fHelp || params.size() > 2) throw runtime_error( "getchaintxstats\n" "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "\nThis RPC will return a large amount of additional data if the shielded index (zindex) is enabled.\n" "\nArguments:\n" "1. nblocks (numeric, optional) Number of blocks in averaging window.\n" "2. blockhash (string, optional) The hash of the block which ends the window.\n" @@ -1965,9 +1970,6 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk int64_t nFullyShieldedPaymentsDiff = pindex->nChainFullyShieldedPayments - pindexPast->nChainFullyShieldedPayments; int64_t nNotarizationsDiff = pindex->nChainNotarizations - pindexPast->nChainNotarizations; - ret.pushKV("window_payments", (int) nPaymentsDiff); - ret.pushKV("window_notarizations", (int) nNotarizationsDiff); - if (nTimeDiff > 0) { ret.pushKV("txrate", ((double)nTxDiff) / nTimeDiff); if (fZindex) { @@ -1981,6 +1983,8 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk } if (fZindex) { + ret.pushKV("window_payments", (int) nPaymentsDiff); + ret.pushKV("window_notarizations", (int) nNotarizationsDiff); ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); @@ -2010,8 +2014,31 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk shielded.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / nShieldedPaymentsDiff ); shielded.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nShieldedPaymentsDiff ); } + if(nShieldedTxDiff+nShieldedPaymentsDiff > 0) ret.pushKV("shielded", shielded); + + // Organic tx stats = Raw - Coinbase - DPoW + if (nTxDiff > 0) { + UniValue organic(UniValue::VOBJ); + + organic.pushKV("shielded_tx_percent", ((double)nShieldedTxDiff) / ORG(nTxDiff)); + organic.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / ORG(nTxDiff)); + organic.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / ORG(nTxDiff)); + organic.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / ORG(nTxDiff)); + organic.pushKV("shielded_payments_percent", ((double)nShieldedPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / ORG(nPaymentsDiff)); + if (nTimeDiff > 0) { + organic.pushKV("paymentrate", ((double)ORG(nPaymentsDiff)) / nTimeDiff); + organic.pushKV("txrate", ((double)ORG(nTxDiff)) / nTimeDiff); + } + organic.pushKV("txcount", (int) ORG(nTxDiff)); + organic.pushKV("payments", (int) ORG(nPaymentsDiff)); + ret.pushKV("organic", organic); + } + } } From 97df4d43ec5e7d3cf272a69df67d98a3a454b47e Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Wed, 18 Dec 2019 13:58:08 -0500 Subject: [PATCH 04/21] Set nChainPayments and increasing debugging/logging --- src/init.cpp | 11 +++++++---- src/main.cpp | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 18c85a3fa..1ec224643 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1755,11 +1755,14 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) boost::filesystem::remove(GetDataDir() / "komodostate"); boost::filesystem::remove(GetDataDir() / "signedmasks"); pblocktree->WriteReindexing(true); + fprintf(stderr, "%s: Deleted komodostate and signedmasks...\n", __FUNCTION__); + //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); } + fprintf(stderr, "%s: Loading block index...\n", __FUNCTION__); if (!LoadBlockIndex()) { strLoadError = _("Error loading block database"); break; @@ -1819,14 +1822,14 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) } if ( KOMODO_REWIND == 0 ) { - if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3), - GetArg("-checkblocks", 288))) { + LogPrintf("Verifying block DB..."); + if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3), GetArg("-checkblocks", 288))) { strLoadError = _("Corrupted block database detected"); break; } } } catch (const std::exception& e) { - if (fDebug) LogPrintf("%s\n", e.what()); + LogPrintf("%s: Error opening block database: %s\n", __FUNCTION__, e.what()); strLoadError = _("Error opening block database"); break; } @@ -1837,7 +1840,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (!fLoaded) { // first suggest a reindex if (!fReset) { - fprintf(stderr,"%s error in hd data\n", __FUNCTION__); + fprintf(stderr,"%s: error in hd data\n", __FUNCTION__); bool fRet = uiInterface.ThreadSafeMessageBox( strLoadError + ".\n\n" + _("error in HDD data, might just need to update to latest, if that doesnt work, then you need to resync"), "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); diff --git a/src/main.cpp b/src/main.cpp index b137a04c9..aa89f7300 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6140,6 +6140,7 @@ bool static LoadBlockIndexDB() pindex->nChainShieldedPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; pindex->nChainShieldingTx = pindex->pprev->nChainShieldingTx + pindex->nShieldingTx; + pindex->nChainPayments = pindex->pprev->nChainPayments + pindex->nPayments; pindex->nChainShieldingPayments = pindex->pprev->nChainShieldingPayments + pindex->nShieldingPayments; pindex->nChainDeshieldingTx = pindex->pprev->nChainShieldedTx + pindex->nShieldedTx; pindex->nChainDeshieldingPayments = pindex->pprev->nChainShieldedPayments + pindex->nShieldedPayments; @@ -6160,6 +6161,7 @@ bool static LoadBlockIndexDB() } else { pindex->nChainTx = 0; if (fZindex) { + pindex->nChainPayments = 0; pindex->nChainNotarizations = 0; pindex->nChainShieldedTx = 0; pindex->nChainFullyShieldedTx = 0; @@ -6179,6 +6181,7 @@ bool static LoadBlockIndexDB() pindex->nChainSproutValue = pindex->nSproutValue; pindex->nChainSaplingValue = pindex->nSaplingValue; if (fZindex) { + pindex->nChainPayments = pindex->nPayments; pindex->nChainNotarizations = pindex->nNotarizations; pindex->nChainShieldedTx = pindex->nShieldedTx; pindex->nChainShieldedPayments = pindex->nShieldedPayments; @@ -6243,7 +6246,7 @@ bool static LoadBlockIndexDB() setBlkDataFiles.insert(pindex->nFile); } } - //fprintf(stderr,"load blockindexDB %u\n",(uint32_t)time(NULL)); + fprintf(stderr,"load blockindexDB %u\n",(uint32_t)time(NULL)); for (std::set::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { CDiskBlockPos pos(*it, 0); From aef3f37b0f2add99ebe5bee6716c73bd3e3928e1 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Wed, 18 Dec 2019 15:20:31 -0500 Subject: [PATCH 05/21] move new zindex stuff to the end of serialization and more logging --- src/chain.h | 29 +++++++++++++++-------------- src/txdb.cpp | 9 ++++++++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/chain.h b/src/chain.h index b596ad75a..2660f6396 100644 --- a/src/chain.h +++ b/src/chain.h @@ -607,20 +607,6 @@ public: READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); - // These values only serialized when -zindex enabled - if((s.GetType() & SER_DISK) && fZindex) { - READWRITE(VARINT(nShieldedTx)); - READWRITE(VARINT(nShieldingTx)); - READWRITE(VARINT(nDeshieldingTx)); - READWRITE(VARINT(nFullyShieldedTx)); - - READWRITE(VARINT(nShieldedPayments)); - READWRITE(VARINT(nShieldingPayments)); - READWRITE(VARINT(nDeshieldingPayments)); - READWRITE(VARINT(nFullyShieldedPayments)); - - READWRITE(VARINT(nNotarizations)); - } if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(nFile)); @@ -668,6 +654,21 @@ public: READWRITE(nNotaryPay); READWRITE(segid); } + + // These values only serialized when -zindex enabled + if((s.GetType() & SER_DISK) && fZindex) { + READWRITE(VARINT(nShieldedTx)); + READWRITE(VARINT(nShieldingTx)); + READWRITE(VARINT(nDeshieldingTx)); + READWRITE(VARINT(nFullyShieldedTx)); + + READWRITE(VARINT(nPayments)); + READWRITE(VARINT(nNotarizations)); + READWRITE(VARINT(nShieldedPayments)); + READWRITE(VARINT(nShieldingPayments)); + READWRITE(VARINT(nDeshieldingPayments)); + READWRITE(VARINT(nFullyShieldedPayments)); + } } uint256 GetBlockHash() const diff --git a/src/txdb.cpp b/src/txdb.cpp index 334977f2d..c1ad0c2d0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -692,18 +692,23 @@ bool CBlockTreeDB::LoadBlockIndexGuts() boost::scoped_ptr pcursor(NewIterator()); pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); + fprintf(stderr,"%s: Seeked cursor to block index\n",__FUNCTION__); // Load mapBlockIndex while (pcursor->Valid()) { + fprintf(stderr,"%s: Valid cursor\n",__FUNCTION__); boost::this_thread::interruption_point(); std::pair key; if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { + fprintf(stderr,"%s: Found DB_BLOCK_INDEX\n",__FUNCTION__); CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object - CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); + fprintf(stderr,"%s: Creating CBlockIndex...\n",__FUNCTION__); + CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->SetHeight(diskindex.GetHeight()); + fprintf(stderr,"%s: Setting CBlockIndex height...\n",__FUNCTION__); pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; @@ -720,8 +725,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nTx = diskindex.nTx; pindexNew->nSproutValue = diskindex.nSproutValue; pindexNew->nSaplingValue = diskindex.nSaplingValue; + fprintf(stderr,"%s: Setting CBlockIndex details...\n",__FUNCTION__); pindexNew->segid = diskindex.segid; pindexNew->nNotaryPay = diskindex.nNotaryPay; + pindexNew->nPayments = diskindex.nPayments; pindexNew->nShieldedTx = diskindex.nShieldedTx; pindexNew->nShieldedPayments = diskindex.nShieldedPayments; pindexNew->nShieldingTx = diskindex.nShieldingTx; From 339f03233d414c4ada5a97b305b4ef8a9c6e6757 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Wed, 18 Dec 2019 15:45:34 -0500 Subject: [PATCH 06/21] Clean up --- src/primitives/block.h | 2 -- src/rpc/misc.cpp | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/primitives/block.h b/src/primitives/block.h index 41e7e3715..d19981def 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -28,8 +28,6 @@ #include "uint256.h" #include "arith_uint256.h" -extern int32_t ASSETCHAINS_LWMAPOS; - /** Nodes collect new transactions into a block, hash them into a hash tree, * and scan through nonce values to make the block's hash satisfy proof-of-work * requirements. When they solve the proof-of-work, they broadcast the block diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 602ec38ec..081938d66 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -83,7 +83,7 @@ extern uint16_t ASSETCHAINS_P2PPORT,ASSETCHAINS_RPCPORT; extern uint32_t ASSETCHAINS_CC; extern uint32_t ASSETCHAINS_MAGIC,ASSETCHAINS_ALGO; extern uint64_t ASSETCHAINS_COMMISSION,ASSETCHAINS_SUPPLY; -extern int32_t ASSETCHAINS_LWMAPOS,ASSETCHAINS_SAPLING,ASSETCHAINS_STAKED; +extern int32_t ASSETCHAINS_SAPLING; extern uint64_t ASSETCHAINS_ENDSUBSIDY[],ASSETCHAINS_REWARD[],ASSETCHAINS_HALVING[],ASSETCHAINS_DECAY[],ASSETCHAINS_NOTARY_PAY[]; extern std::string NOTARY_PUBKEY,NOTARY_ADDRESS; extern uint8_t NOTARY_PUBKEY33[]; @@ -348,10 +348,6 @@ UniValue getinfo(const UniValue& params, bool fHelp, const CPubKey& mypk) if ( ASSETCHAINS_COMMISSION != 0 ) obj.push_back(Pair("commission", ASSETCHAINS_COMMISSION)); - if ( ASSETCHAINS_STAKED != 0 ) - obj.push_back(Pair("staked", ASSETCHAINS_STAKED)); - if ( ASSETCHAINS_LWMAPOS != 0 ) - obj.push_back(Pair("lwmapos", ASSETCHAINS_LWMAPOS)); if ( ASSETCHAINS_ALGO != ASSETCHAINS_EQUIHASH ) obj.push_back(Pair("algo",ASSETCHAINS_ALGORITHMS[ASSETCHAINS_ALGO])); } From 12e7c13a84c78649f23a324a66c41acfe1e5b756 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 00:10:52 -0500 Subject: [PATCH 07/21] Change datatypes, do not use VARINT, increase exception logging --- src/chain.h | 47 ++++++++++++++++++++++++++------------------- src/dbwrapper.h | 5 ++++- src/txdb.cpp | 2 ++ src/wallet/wallet.h | 1 + 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/chain.h b/src/chain.h index 2660f6396..85d41b47b 100644 --- a/src/chain.h +++ b/src/chain.h @@ -262,49 +262,49 @@ public: unsigned int nTx; //! Number of notarization transactions in this block. - unsigned int nNotarizations; + int64_t nNotarizations; //! (memory only) Number of payments (shielded or transparent) in the block //! up to and including this block. One transaction can contain one or more //! payments. This stat allows us to calculate ratios of shielded/transparent //! when combined with shielded payment stats - unsigned int nPayments; + int64_t nPayments; //! (memory only) Number of shielded transactions (of any kind) in the block up to and including this block. //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. - unsigned int nShieldedTx; + int64_t nShieldedTx; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... - unsigned int nFullyShieldedTx; + int64_t nFullyShieldedTx; //! (memory only) Number of shielding payments. A shielding payment is defined //! as having a shielded output but transparent input: t->z - unsigned int nShieldingPayments; + int64_t nShieldingPayments; //! (memory only) Number of shielded payments. A shielded payment is defined //! as having a shielded input or output: t->z or z->t - unsigned int nShieldedPayments; + int64_t nShieldedPayments; //! (memory only) Number of fully shielded payments. A fully shielded payment is defined //! as having a shielded input and shielded output: z->z - unsigned int nFullyShieldedPayments; + int64_t nFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent output. - unsigned int nDeshieldingTx; + int64_t nDeshieldingTx; //! (memory only) Number of deshielding payments. A deshielding payment is defined //! as one transparent input and one shielded output: z->t - unsigned int nDeshieldingPayments; + int64_t nDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined //! as a transaction containing JoinSplits and at least one transparent input // i.e. t->z or t->(z,t) or z->(z,z,t) - unsigned int nShieldingTx; + int64_t nShieldingTx; //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. @@ -414,6 +414,7 @@ public: chainPower = CChainPower(); nTx = 0; nChainTx = 0; + nChainPayments = 0; nChainShieldedTx = 0; nChainShieldingTx = 0; @@ -424,14 +425,18 @@ public: nChainShieldingPayments = 0; nChainDeshieldingPayments = 0; nChainFullyShieldedPayments = 0; + + nPayments = 0; nShieldedTx = 0; nShieldingTx = 0; + nNotarizations = 0; nDeshieldingTx = 0; nFullyShieldedTx = 0; nShieldedPayments = 0; nShieldingPayments = 0; nDeshieldingPayments = 0; nFullyShieldedPayments = 0; + nStatus = 0; nCachedBranchId = boost::none; hashSproutAnchor = uint256(); @@ -649,25 +654,27 @@ public: if ((s.GetType() & SER_DISK) && (nVersion >= SAPLING_VALUE_VERSION)) { READWRITE(nSaplingValue); } + /* if ( (s.GetType() & SER_DISK) && (is_STAKED(ASSETCHAINS_SYMBOL) != 0) && ASSETCHAINS_NOTARY_PAY[0] != 0 ) { READWRITE(nNotaryPay); READWRITE(segid); } + */ // These values only serialized when -zindex enabled if((s.GetType() & SER_DISK) && fZindex) { - READWRITE(VARINT(nShieldedTx)); - READWRITE(VARINT(nShieldingTx)); - READWRITE(VARINT(nDeshieldingTx)); - READWRITE(VARINT(nFullyShieldedTx)); + READWRITE(nShieldedTx); + READWRITE(nShieldingTx); + READWRITE(nDeshieldingTx); + READWRITE(nFullyShieldedTx); - READWRITE(VARINT(nPayments)); - READWRITE(VARINT(nNotarizations)); - READWRITE(VARINT(nShieldedPayments)); - READWRITE(VARINT(nShieldingPayments)); - READWRITE(VARINT(nDeshieldingPayments)); - READWRITE(VARINT(nFullyShieldedPayments)); + READWRITE(nPayments); + READWRITE(nNotarizations); + READWRITE(nShieldedPayments); + READWRITE(nShieldingPayments); + READWRITE(nDeshieldingPayments); + READWRITE(nFullyShieldedPayments); } } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 21f57d094..a43383a0a 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,4 +1,5 @@ // Copyright (c) 2012-2014 The Bitcoin Core developers +// Copyright (c) 2019 The Hush developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -143,6 +144,7 @@ public: CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); ssValue >> value; } catch(std::exception &e) { + LogPrintf("%s: CDataStream error - %s\n", __FUNCTION__, e.what()); return false; } return true; @@ -207,7 +209,8 @@ public: try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); ssValue >> value; - } catch (const std::exception&) { + } catch (const std::exception &e) { + LogPrintf("%s: CDataStream error - %s\n", __FUNCTION__, e.what()); return false; } return true; diff --git a/src/txdb.cpp b/src/txdb.cpp index c1ad0c2d0..536a7b7f8 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -287,10 +287,12 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); + fprintf(stderr, "%s: Writing block files\n", __FUNCTION__); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } batch.Write(DB_LAST_BLOCK, nLastFile); + fprintf(stderr, "%s: Writing block index\n", __FUNCTION__); for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 62b003f98..07b80cd6a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1,5 +1,6 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2019 The Hush developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. From 9c3d8465733ae83fb6c2453b9bb996b37687183b Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 09:00:37 -0500 Subject: [PATCH 08/21] Fix rpc c++ tests --- src/test/rpc_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 7cb1ef010..d0e94ab5d 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -46,7 +46,7 @@ UniValue CallRPC(string args) BOOST_CHECK(tableRPC[strMethod]); rpcfn_type method = tableRPC[strMethod]->actor; try { - UniValue result = (*method)(params, false); + UniValue result = (*method)(params, false, CPubKey()); return result; } catch (const UniValue& objError) { From 284f8a7e607aa7a187e487db58ecdf709abde370 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 09:18:16 -0500 Subject: [PATCH 09/21] Pull in test change from upstream --- src/test-komodo/test_eval_notarisation.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test-komodo/test_eval_notarisation.cpp b/src/test-komodo/test_eval_notarisation.cpp index 0abac499d..f0c48f951 100644 --- a/src/test-komodo/test_eval_notarisation.cpp +++ b/src/test-komodo/test_eval_notarisation.cpp @@ -56,7 +56,8 @@ namespace TestEvalNotarisation { } }; - static auto noop = [&](CMutableTransaction &mtx){}; + //static auto noop = [&](CMutableTransaction &mtx){}; + static auto noop = [](CMutableTransaction &mtx){}; template From b52d9786a65ee37b327913e56ae29a8e1682c7ff Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 09:24:52 -0500 Subject: [PATCH 10/21] Update some test files --- qa/pull-tester/run-bitcoind-for-test.sh.in | 14 ++++++++------ qa/pull-tester/tests-config.sh.in | 6 ++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/qa/pull-tester/run-bitcoind-for-test.sh.in b/qa/pull-tester/run-bitcoind-for-test.sh.in index 999b0f5b7..afd45216d 100755 --- a/qa/pull-tester/run-bitcoind-for-test.sh.in +++ b/qa/pull-tester/run-bitcoind-for-test.sh.in @@ -1,22 +1,24 @@ #!/bin/bash +# THIS FILE IS GENERATED FROM run-bitcoind-for-test.sh.in # Copyright (c) 2013-2014 The Bitcoin Core developers +# Copyright (c) 2019 The Hush developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -ZCASH_LOAD_TIMEOUT=500 -DATADIR="@abs_top_builddir@/.zcash" +HUSH_LOAD_TIMEOUT=500 +DATADIR="@abs_top_builddir@/.komodo/HUSH3" rm -rf "$DATADIR" mkdir -p "$DATADIR"/regtest -touch "$DATADIR/zcash.conf" +touch "$DATADIR/HUSH3.conf" touch "$DATADIR/regtest/debug.log" tail -q -n 1 -F "$DATADIR/regtest/debug.log" | grep -m 1 -q "Done loading" & WAITER=$! PORT=`expr 10000 + $$ % 55536` -"@abs_top_builddir@/src/zcashd@EXEEXT@" -connect=0.0.0.0 -datadir="$DATADIR" -rpcuser=user -rpcpassword=pass -listen -keypool=3 -debug -debug=net -logtimestamps -checkmempool=0 -relaypriority=0 -port=$PORT -whitelist=127.0.0.1 -regtest -rpcport=`expr $PORT + 1` & +"@abs_top_builddir@/src/hushd@EXEEXT@" -connect=0.0.0.0 -datadir="$DATADIR" -rpcuser=user -rpcpassword=pass -listen -keypool=3 -debug -debug=net -logtimestamps -checkmempool=0 -relaypriority=0 -port=$PORT -whitelist=127.0.0.1 -regtest -rpcport=`expr $PORT + 1` & BITCOIND=$! #Install a watchdog. -(sleep "$ZCASH_LOAD_TIMEOUT" && kill -0 $WAITER 2>/dev/null && kill -9 $BITCOIND $$)& +(sleep "$HUSH_LOAD_TIMEOUT" && kill -0 $WAITER 2>/dev/null && kill -9 $BITCOIND $$)& wait $WAITER if [ -n "$TIMEOUT" ]; then @@ -27,7 +29,7 @@ else RETURN=$? fi -(sleep "$ZCASH_LOAD_TIMEOUT" && kill -0 $BITCOIND 2>/dev/null && kill -9 $BITCOIND $$)& +(sleep "$HUSH_LOAD_TIMEOUT" && kill -0 $BITCOIND 2>/dev/null && kill -9 $BITCOIND $$)& kill $BITCOIND && wait $BITCOIND # timeout returns 124 on timeout, otherwise the return value of the child diff --git a/qa/pull-tester/tests-config.sh.in b/qa/pull-tester/tests-config.sh.in index 8d9467f3f..8f2cf3582 100755 --- a/qa/pull-tester/tests-config.sh.in +++ b/qa/pull-tester/tests-config.sh.in @@ -1,5 +1,7 @@ #!/bin/bash +# THIS FILE IS GENERATED FROM tests-config.sh.in # Copyright (c) 2013-2014 The Bitcoin Core developers +# Copyright (c) 2019 The Hush developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +15,6 @@ EXEEXT="@EXEEXT@" @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1 @ENABLE_PROTON_TRUE@ENABLE_PROTON=1 -REAL_BITCOIND="$BUILDDIR/src/komodod${EXEEXT}" -REAL_BITCOINCLI="$BUILDDIR/src/komodo-cli${EXEEXT}" +REAL_BITCOIND="$BUILDDIR/src/hushd${EXEEXT}" +REAL_BITCOINCLI="$BUILDDIR/src/hush-cli${EXEEXT}" From bb0b661009d3ff0975d2644ec593ffe7cb775342 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 09:28:47 -0500 Subject: [PATCH 11/21] Update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 965088922..c28dba409 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ Makefile.in src/libcc.so src/cc/customcc.so src/HUSH3_7776 +src/cc/librogue.so +src/cc/games/prices +src/cc/games/tetris From 0baa4ecadfaaab424590d74e928a1a534717b568 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 10:00:04 -0500 Subject: [PATCH 12/21] Guard against negative values in statistics and add some rpc docs --- src/rpc/blockchain.cpp | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d4fe63f8b..feb53be62 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1875,8 +1875,8 @@ inline CBlockIndex* LookupBlockIndex(const uint256& hash) } // given a transaction count X, subtract out coinbase and dpow transactions -// to give an "organic count" -#define ORG(X) (X - blockcount - nNotarizationsDiff) +// to give an "organic count". We return 0 instead of negative values +#define ORG(X) ( (X - blockcount - nNotarizationsDiff) > 0 ? (X - blockcount - nNotarizationsDiff) : 0 ) UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk) { @@ -1892,7 +1892,12 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk "{\n" " \"time\": xxxxx, (numeric) The timestamp for the final block in the window in UNIX format.\n" " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" - " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" + " \"shielded_txcount\": xxxxx, (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n" + " \"shielding_txcount\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n" + " \"deshielding_txcount\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n" + " \"notarizations\": xxxxx, (numeric) The total number of notarization transactions in the chain up to that point.\n" + " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" + " \"window_final_block_height\": xxxxx, (numeric) Block height of final block in window.\n" " \"window_block_count\": xxxxx, (numeric) Size of the window in number of blocks.\n" " \"window_tx_count\": xxxxx, (numeric) The number of transactions in the window. Only returned if \"window_block_count\" is > 0.\n" " \"window_interval\": xxxxx, (numeric) The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0.\n" @@ -2022,15 +2027,19 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk if (nTxDiff > 0) { UniValue organic(UniValue::VOBJ); - organic.pushKV("shielded_tx_percent", ((double)nShieldedTxDiff) / ORG(nTxDiff)); - organic.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / ORG(nTxDiff)); - organic.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / ORG(nTxDiff)); - organic.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / ORG(nTxDiff)); - organic.pushKV("shielded_payments_percent", ((double)nShieldedPaymentsDiff) / ORG(nPaymentsDiff)); - organic.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / ORG(nPaymentsDiff)); - organic.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / ORG(nPaymentsDiff)); - organic.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / ORG(nPaymentsDiff)); - if (nTimeDiff > 0) { + if(ORG(nTxDiff) > 0) { + organic.pushKV("shielded_tx_percent", ((double)nShieldedTxDiff) / ORG(nTxDiff)); + organic.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / ORG(nTxDiff)); + organic.pushKV("shielding_tx_percent", ((double)nShieldingTxDiff) / ORG(nTxDiff)); + organic.pushKV("deshielding_tx_percent", ((double)nDeshieldingTxDiff) / ORG(nTxDiff)); + } + if(ORG(nPaymentsDiff) > 0) { + organic.pushKV("shielded_payments_percent", ((double)nShieldedPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("fully_shielded_payments_percent", ((double)nFullyShieldedPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("shielding_payments_percent", ((double)nShieldingPaymentsDiff) / ORG(nPaymentsDiff)); + organic.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / ORG(nPaymentsDiff)); + } + if (nTimeDiff > 0) { organic.pushKV("paymentrate", ((double)ORG(nPaymentsDiff)) / nTimeDiff); organic.pushKV("txrate", ((double)ORG(nTxDiff)) / nTimeDiff); } From bf75e6aea391416a2c44fd652e932504bdd97087 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 10:02:24 -0500 Subject: [PATCH 13/21] Add payment rpc docs for getchaintxstats --- src/rpc/blockchain.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index feb53be62..8c93a074f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1895,6 +1895,12 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk " \"shielded_txcount\": xxxxx, (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n" " \"shielding_txcount\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n" " \"deshielding_txcount\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n" + " \"fully_shielded_txcount\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) transactions in the chain up to that point.\n" + " \"payments\": xxxxx, (numeric) The total number of payments in the chain up to that point.\n" + " \"shielded_payments\": xxxxx, (numeric) The total number of shielded (containing a zaddr) payments in the chain up to that point.\n" + " \"shielding_payments\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) payments in the chain up to that point.\n" + " \"deshielding_payments\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) payments in the chain up to that point.\n" + " \"fully_shielded_payments\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) payments in the chain up to that point.\n" " \"notarizations\": xxxxx, (numeric) The total number of notarization transactions in the chain up to that point.\n" " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" " \"window_final_block_height\": xxxxx, (numeric) Block height of final block in window.\n" From 284edb0968fe7f9afee8326a1bd9272842c2afb0 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 12:38:06 -0500 Subject: [PATCH 14/21] Add some forgotten zstats that are important --- src/rpc/blockchain.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8c93a074f..bfbd2dd84 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1996,6 +1996,10 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk if (fZindex) { ret.pushKV("window_payments", (int) nPaymentsDiff); ret.pushKV("window_notarizations", (int) nNotarizationsDiff); + ret.pushKV("window_fully_shielded_txcount", nFullyShieldedTxDiff); + ret.pushKV("window_deshielding_txcount", nDeshieldingTxDiff); + ret.pushKV("window_shielding_txcount", nShieldingTxDiff); + ret.pushKV("window_shielded_txcount", nShieldedTxDiff); ret.pushKV("window_fully_shielded_payments", nFullyShieldedPaymentsDiff); ret.pushKV("window_shielded_payments", nShieldedPaymentsDiff); ret.pushKV("window_shielding_payments", nShieldingPaymentsDiff); @@ -2013,7 +2017,7 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk ret.pushKV("deshielding_payments_percent", ((double)nDeshieldingPaymentsDiff) / nPaymentsDiff); } - // Shielded statistics + // Statistics considering only zxtns UniValue shielded(UniValue::VOBJ); if (nShieldedTxDiff > 0) { shielded.pushKV("fully_shielded_tx_percent", ((double)nFullyShieldedTxDiff) / nShieldedTxDiff ); From feee580c33bbd7ac3c47996d9076bf1e51aafe7a Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 16:10:29 -0500 Subject: [PATCH 15/21] Add more getchaintxstats rpc docs --- src/rpc/blockchain.cpp | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index bfbd2dd84..138b4dac9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1890,24 +1890,32 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk "2. blockhash (string, optional) The hash of the block which ends the window.\n" "\nResult:\n" "{\n" - " \"time\": xxxxx, (numeric) The timestamp for the final block in the window in UNIX format.\n" - " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" - " \"shielded_txcount\": xxxxx, (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n" - " \"shielding_txcount\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n" - " \"deshielding_txcount\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n" - " \"fully_shielded_txcount\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) transactions in the chain up to that point.\n" - " \"payments\": xxxxx, (numeric) The total number of payments in the chain up to that point.\n" - " \"shielded_payments\": xxxxx, (numeric) The total number of shielded (containing a zaddr) payments in the chain up to that point.\n" - " \"shielding_payments\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) payments in the chain up to that point.\n" - " \"deshielding_payments\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) payments in the chain up to that point.\n" - " \"fully_shielded_payments\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) payments in the chain up to that point.\n" - " \"notarizations\": xxxxx, (numeric) The total number of notarization transactions in the chain up to that point.\n" - " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" - " \"window_final_block_height\": xxxxx, (numeric) Block height of final block in window.\n" - " \"window_block_count\": xxxxx, (numeric) Size of the window in number of blocks.\n" - " \"window_tx_count\": xxxxx, (numeric) The number of transactions in the window. Only returned if \"window_block_count\" is > 0.\n" - " \"window_interval\": xxxxx, (numeric) The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0.\n" - " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0.\n" + " \"time\": xxxxx, (numeric) The timestamp for the final block in the window in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"shielded_txcount\": xxxxx, (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n" + " \"shielding_txcount\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n" + " \"deshielding_txcount\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n" + " \"fully_shielded_txcount\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) transactions in the chain up to that point.\n" + " \"payments\": xxxxx, (numeric) The total number of payments in the chain up to that point.\n" + " \"shielded_payments\": xxxxx, (numeric) The total number of shielded (containing a zaddr) payments in the chain up to that point.\n" + " \"shielding_payments\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) payments in the chain up to that point.\n" + " \"deshielding_payments\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) payments in the chain up to that point.\n" + " \"fully_shielded_payments\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) payments in the chain up to that point.\n" + " \"notarizations\": xxxxx, (numeric) The total number of notarization transactions in the chain up to that point.\n" + " \"window_final_block_hash\": \"...\", (string) The hash of the final block in the window.\n" + " \"window_final_block_height\": xxxxx, (numeric) Block height of final block in window.\n" + " \"window_block_count\": xxxxx, (numeric) Size of the window in number of blocks.\n" + " \"window_tx_count\": xxxxx, (numeric) The number of transactions in the window. Only returned if \"window_block_count\" is > 0.\n" + " \"window_interval\": xxxxx, (numeric) The elapsed time in the window in seconds. Only returned if \"window_block_count\" is > 0.\n" + " \"window_shielded_txcount\": xxxxx, (numeric) The total number of shielded (containing a zaddr) transactions in the chain up to that point.\n" + " \"window_shielding_txcount\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) transactions in the chain up to that point.\n" + " \"window_deshielding_txcount\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) transactions in the chain up to that point.\n" + " \"window_fully_shielded_txcount\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) transactions in the chain up to that point.\n" + " \"window_shielded_payments\": xxxxx, (numeric) The total number of shielded (containing a zaddr) payments in the chain up to that point.\n" + " \"window_shielding_payments\": xxxxx, (numeric) The total number of shielding (containing a zaddr output) payments in the chain up to that point.\n" + " \"window_deshielding_payments\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) payments in the chain up to that point.\n" + " \"window_fully_shielded_payments\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) payments in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0.\n" "}\n" "\nExamples:\n" + HelpExampleCli("getchaintxstats", "") From 55258a54dc2314ba3aac04fac1ac30e401aecd73 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 16:22:42 -0500 Subject: [PATCH 16/21] Add rpc docs about shielded-only stats --- src/rpc/blockchain.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 138b4dac9..d9a9d221f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1916,6 +1916,14 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk " \"window_deshielding_payments\": xxxxx, (numeric) The total number of deshielding (containing a zaddr input) payments in the chain up to that point.\n" " \"window_fully_shielded_payments\": xxxxx, (numeric) The total number of z2z, AKA fully-shielded (containing only zaddr inputs+outputs) payments in the chain up to that point.\n" " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window. Only returned if \"window_interval\" is > 0.\n" + " \"shielded\": { (string) The set of stats specific to only shieled transactions. \n" + " \"fully_shielded_tx_percent\": (numeric) The percentage of fully shielded transactions.\n" + " \"shielding_tx_percent\": (numeric) The percentage of shielding transactions.\n" + " \"deshielding_tx_percent\": (numeric) The percentage of deshielding transactions.\n" + " \"fully_shielded_payments_percent\": (numeric) The percentage of fully shielded payments.\n" + " \"shielding_payments_percent\": (numeric) The percentage of shielding payments.\n" + " \"deshielding_payments_percent\": (numeric) The percentage of deshielding payments.\n" + " },\n" "}\n" "\nExamples:\n" + HelpExampleCli("getchaintxstats", "") From dbf1c23a81d17e85f31ae8b09547c4b6bae066c5 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 17:25:07 -0500 Subject: [PATCH 17/21] Add rpc docs about organic xtns --- src/rpc/blockchain.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d9a9d221f..90cab6870 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1924,6 +1924,14 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk " \"shielding_payments_percent\": (numeric) The percentage of shielding payments.\n" " \"deshielding_payments_percent\": (numeric) The percentage of deshielding payments.\n" " },\n" + " \"organic\": { (string) The set of stats about organic transactions, i.e. those that are not coinbase and not notarizations\n" + " \"fully_shielded_tx_percent\": (numeric) The percentage of fully shielded organic transactions.\n" + " \"shielding_tx_percent\": (numeric) The percentage of shielding organic transactions.\n" + " \"deshielding_tx_percent\": (numeric) The percentage of deshielding organic transactions.\n" + " \"fully_shielded_payments_percent\": (numeric) The percentage of fully shielded organic payments.\n" + " \"shielding_payments_percent\": (numeric) The percentage of shielding organic payments.\n" + " \"deshielding_payments_percent\": (numeric) The percentage of deshielding organic payments.\n" + " }\n" "}\n" "\nExamples:\n" + HelpExampleCli("getchaintxstats", "") From a2e63618e4b87c1572685c9dd89ce873414cd225 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Thu, 19 Dec 2019 17:46:58 -0500 Subject: [PATCH 18/21] Tone down our STDOUT debugging --- src/txdb.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 536a7b7f8..889c3f14a 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -694,23 +694,23 @@ bool CBlockTreeDB::LoadBlockIndexGuts() boost::scoped_ptr pcursor(NewIterator()); pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); - fprintf(stderr,"%s: Seeked cursor to block index\n",__FUNCTION__); + //fprintf(stderr,"%s: Seeked cursor to block index\n",__FUNCTION__); // Load mapBlockIndex while (pcursor->Valid()) { - fprintf(stderr,"%s: Valid cursor\n",__FUNCTION__); + //fprintf(stderr,"%s: Valid cursor\n",__FUNCTION__); boost::this_thread::interruption_point(); std::pair key; if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { - fprintf(stderr,"%s: Found DB_BLOCK_INDEX\n",__FUNCTION__); + //fprintf(stderr,"%s: Found DB_BLOCK_INDEX\n",__FUNCTION__); CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object - fprintf(stderr,"%s: Creating CBlockIndex...\n",__FUNCTION__); + //fprintf(stderr,"%s: Creating CBlockIndex...\n",__FUNCTION__); CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->SetHeight(diskindex.GetHeight()); - fprintf(stderr,"%s: Setting CBlockIndex height...\n",__FUNCTION__); + //fprintf(stderr,"%s: Setting CBlockIndex height...\n",__FUNCTION__); pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; @@ -727,7 +727,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nTx = diskindex.nTx; pindexNew->nSproutValue = diskindex.nSproutValue; pindexNew->nSaplingValue = diskindex.nSaplingValue; - fprintf(stderr,"%s: Setting CBlockIndex details...\n",__FUNCTION__); + //fprintf(stderr,"%s: Setting CBlockIndex details...\n",__FUNCTION__); pindexNew->segid = diskindex.segid; pindexNew->nNotaryPay = diskindex.nNotaryPay; pindexNew->nPayments = diskindex.nPayments; @@ -741,7 +741,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nFullyShieldedPayments = diskindex.nFullyShieldedPayments; pindexNew->nNotarizations = diskindex.nNotarizations; - fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); + //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks auto header = pindexNew->GetBlockHeader(); if (header.GetHash() != pindexNew->GetBlockHash()) From 80868ef5e44b4652fd0eac53e97c0e692e95ec54 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Fri, 20 Dec 2019 09:05:08 -0500 Subject: [PATCH 19/21] Make sure to count taddr outputs in z=>t xtns as shielded payments --- src/main.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index aa89f7300..26c31f07b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4758,12 +4758,12 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } //NOTE: These are at best heuristics. Improve them as much as possible. // You cannot compare stats generated from different sets of heuristics, so - // if you change this code, you must delete and resync from scratch, or you + // if you change this code, you must reindex or delete and resync from scratch, or you // will be mixing together data from two set of heuristics. if (nShieldedOutputs >= 1) { // If there are shielded outputs, count each as a payment - // By default, if there is more than 1 output, we assume 1 change output which is not a payment. + // By default, if there is more than 1 output, we assume 1 zaddr change output which is not a payment. // In the case of multiple outputs which spend inputs exactly, there is no change output and this // heuristic will undercount payments. Since this edge case is rare, this seems acceptable. // t->(t,t,z) = 1 shielded payment @@ -4772,7 +4772,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // t->(t,z) = 1 shielded payment + transparent change // (z,z)->z = 1 shielded payment (has this xtn ever occurred?) // z->(z,z,z) = 2 shielded payments + shielded change - // Assume that there is always 1 change output when there are more than one + // Assume that there is always 1 change output when there are more than one output nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; // Fully shielded do not count toward shielding/deshielding @@ -4780,6 +4780,8 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nFullyShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; } else { nShieldingPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + // Also count remaining taddr outputs as payments + nShieldedPayments += tx.vout.size(); } } else if (nShieldedSpends >=1) { // Shielded inputs with no shielded outputs. We know none are change output because @@ -4790,7 +4792,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedPayments += tx.vout.size(); nDeshieldingPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); } - //TODO: correctly add shielded payments to total chain payments + //TODO: correctly add transparent payments nPayments += nShieldedPayments; } else { // No shielded payments, add transparent payments minus a change address @@ -4833,7 +4835,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if (fZindex) { - fprintf(stderr,"%s: setting chain zstats\n", __FUNCTION__); + //fprintf(stderr,"%s: setting chain zstats\n", __FUNCTION__); pindex->nChainNotarizations = (pindex->pprev ? pindex->pprev->nChainNotarizations : 0) + pindex->nNotarizations; pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; pindex->nChainFullyShieldedTx = (pindex->pprev ? pindex->pprev->nChainFullyShieldedTx : 0) + pindex->nFullyShieldedTx; From 2620de459fb14395a02d57bdae070f9ca0c8a2f8 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Fri, 20 Dec 2019 10:47:24 -0500 Subject: [PATCH 20/21] Correctly count taddr outputs towards shielded payments --- src/main.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 26c31f07b..4f0efd90b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4743,12 +4743,22 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl isShieldedTx = (nShieldedSpends + nShieldedOutputs) > 0 ? true : false; // We want to avoid full verification with a low false-positive rate + // TODO: A nefarious user could create xtns which meet these criteria and skew stats, what + // else can we look for which is not full validation? + // Can we filter on properties of tx.vout[0] ? if(tx.vin.size()==13 && tx.vout.size()==2 && tx.vout[1].scriptPubKey.IsOpReturn() && tx.vout[1].nValue==0) { nNotarizations++; } + //NOTE: These are at best heuristics. Improve them as much as possible. + // You cannot compare stats generated from different sets of heuristics, so + // if you change this code, you must reindex or delete datadir + resync from scratch, or you + // will be mixing together data from two set of heuristics. if(isShieldedTx) { nShieldedTx++; + // NOTE: It's possible for very complex transactions to be both shielding and deshielding, + // such as (t,z)=>(t,z) Since these transactions cannot be made via RPCs currently, they + // would currently need to be made via raw transactions if(tx.vin.size()==0 && tx.vout.size()==0) { nFullyShieldedTx++; } else if(tx.vin.size()>0) { @@ -4756,10 +4766,6 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } else if(tx.vout.size()>0) { nDeshieldingTx++; } - //NOTE: These are at best heuristics. Improve them as much as possible. - // You cannot compare stats generated from different sets of heuristics, so - // if you change this code, you must reindex or delete and resync from scratch, or you - // will be mixing together data from two set of heuristics. if (nShieldedOutputs >= 1) { // If there are shielded outputs, count each as a payment @@ -4773,7 +4779,9 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl // (z,z)->z = 1 shielded payment (has this xtn ever occurred?) // z->(z,z,z) = 2 shielded payments + shielded change // Assume that there is always 1 change output when there are more than one output - nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + nShieldedPayments += nShieldedOutputs > 1 ? (nShieldedOutputs-1) : 1; + // since we have at least 1 zoutput, all transparent outputs are payments, not change + nShieldedPayments += tx.vout.size(); // Fully shielded do not count toward shielding/deshielding if(tx.vin.size()==0 && tx.vout.size()==0) { @@ -4792,7 +4800,6 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl nShieldedPayments += tx.vout.size(); nDeshieldingPayments += tx.vout.size() > 1 ? tx.vout.size()-1 : tx.vout.size(); } - //TODO: correctly add transparent payments nPayments += nShieldedPayments; } else { // No shielded payments, add transparent payments minus a change address From 08361bec95f166a6ac466f45fab4ec4f35c601a4 Mon Sep 17 00:00:00 2001 From: Duke Leto Date: Fri, 20 Dec 2019 11:40:43 -0500 Subject: [PATCH 21/21] Prevent garbage data during chain sync --- src/rpc/blockchain.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 90cab6870..ffbc74e63 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1878,8 +1878,13 @@ inline CBlockIndex* LookupBlockIndex(const uint256& hash) // to give an "organic count". We return 0 instead of negative values #define ORG(X) ( (X - blockcount - nNotarizationsDiff) > 0 ? (X - blockcount - nNotarizationsDiff) : 0 ) +//TODO: Allow custom error message in this macro +#define THROW_IF_SYNCING(INSYNC) if (INSYNC == 0) { throw runtime_error(strprintf("%s: Chain still syncing at height %d, aborting to prevent garbage data. Please wait until the chain is synced to run this RPC",__FUNCTION__,chainActive.Tip()->GetHeight())); } + UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk) { + THROW_IF_SYNCING(KOMODO_INSYNC); + if (fHelp || params.size() > 2) throw runtime_error( "getchaintxstats\n"