diff --git a/src/chain.h b/src/chain.h index 1d8f2a462..bb0b2897f 100644 --- a/src/chain.h +++ b/src/chain.h @@ -266,7 +266,7 @@ public: int64_t nPayments; //! (memory only) Number of shielded transactions (of any kind) in the block up to and including this block. - //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! A shielded transaction is defined as a transaction that contains at least 1 ShieldedInput or ShieldedOutput //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. int64_t nShieldedTx; @@ -278,7 +278,7 @@ public: int64_t nShieldedSpends; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined - //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + //! as a transaction containing only shielded inputs and outputs, i.e. no transparent // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... int64_t nFullyShieldedTx; @@ -295,7 +295,7 @@ public: int64_t nFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined - //! as a transaction containing JoinSplits and at least one transparent output. + //! as a transaction containing ShieldedInputs and at least one transparent output. int64_t nDeshieldingTx; //! (memory only) Number of deshielding payments. A deshielding payment is defined @@ -303,7 +303,7 @@ public: int64_t nDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined - //! as a transaction containing JoinSplits and at least one transparent input + //! as a transaction containing ShieldedOutputs and at least one transparent input // i.e. t->z or t->(z,t) or z->(z,z,t) int64_t nShieldingTx; @@ -322,7 +322,7 @@ public: int64_t nChainPayments; //! (memory only) Number of shielded transactions (of any kind) in the chain up to and including this block. - //! A shielded transaction is defined as a transaction that contains at least 1 JoinSplit, which includes + //! A shielded transaction is defined as a transaction that contains at least 1 ShieldedInput or ShieldedOutput //! shielding/de-shielding and other complex transaction possibilties including multiple taddrs/zaddrs as //! inputs and outputs. int64_t nChainShieldedTx; @@ -334,7 +334,7 @@ public: int64_t nChainShieldedSpends; //! (memory only) Number of fully shielded transactions. A fully shielded transaction is defined - //! as a transaction containing JoinSplits and only shielded inputs and outputs, i.e. no transparent + //! as a transaction containing and only shielded inputs and outputs, i.e. no transparent // inputs or outputs: z->z or z->(z,z) or z->(z,z,z,) etc... int64_t nChainFullyShieldedTx; @@ -351,7 +351,7 @@ public: int64_t nChainFullyShieldedPayments; //! (memory only) Number of deshielding transactions. A deshielding transaction is defined - //! as a transaction containing JoinSplits and at least one transparent output. + //! as a transaction containing ShieldedInputs and at least one transparent output. int64_t nChainDeshieldingTx; //! (memory only) Number of deshielding payments. A deshielding payment is defined @@ -359,7 +359,7 @@ public: int64_t nChainDeshieldingPayments; //! (memory only) Number of shielding transactions. A shielding transaction is defined - //! as a transaction containing JoinSplits and at least one transparent input + //! as a transaction containing ShieldedOutputs and at least one transparent input // i.e. t->z or t->(z,t) or z->(z,z,t) int64_t nChainShieldingTx; diff --git a/src/main.cpp b/src/main.cpp index 2d352d12d..94b41fe5a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -76,6 +76,7 @@ using namespace std; CCriticalSection cs_main; extern uint8_t NOTARY_PUBKEY33[33]; extern int32_t HUSH_LOADINGBLOCKS,HUSH_LONGESTCHAIN,HUSH_INSYNC,HUSH_CONNECTING,HUSH_EXTRASATOSHI; +extern CZindexStats zstats; int32_t HUSH_NEWBLOCKS; int32_t hush_block2pubkey33(uint8_t *pubkey33,CBlock *block); bool Getscriptaddress(char *destaddr,const CScript &scriptPubKey); @@ -572,6 +573,102 @@ namespace { } // anon namespace +// CZindexDB +CZindexDB::CZindexDB() +{ + pathAddr = GetDataDir() / "zindex.dat"; +} + +bool CZindexDB::Read(CZindexStats& zstats) +{ + // open input file, and associate with CAutoFile + FILE *file = fopen(pathAddr.string().c_str(), "rb"); + CAutoFile filein(file, SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("%s: Failed to open file %s", __func__, pathAddr.string()); + + // use file size to size memory buffer + int fileSize = boost::filesystem::file_size(pathAddr); + int dataSize = fileSize - sizeof(uint256); + // Don't try to resize to a negative number if file is small + if (dataSize < 0) + dataSize = 0; + vector vchData; + vchData.resize(dataSize); + uint256 hashIn; + + // read data and checksum from file + try { + filein.read((char *)&vchData[0], dataSize); + filein >> hashIn; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + filein.fclose(); + + CDataStream ssZstats(vchData, SER_DISK, CLIENT_VERSION); + + // verify stored checksum matches input data + uint256 hashTmp = Hash(ssZstats.begin(), ssZstats.end()); + if (hashIn != hashTmp) + return error("%s: zstats Checksum mismatch, data corrupted", __func__); + + unsigned char pchMsgTmp[4]; + try { + // de-serialize file header (network specific magic number) and .. + ssZstats >> FLATDATA(pchMsgTmp); + + // ... verify the network matches ours + if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) + return error("%s: Invalid network magic number", __func__); + + // de-serialize data into one CZindexStats object + ssZstats >> zstats; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + return true; +} + +bool CZindexDB::Write(const CZindexStats& zstats) +{ + // Generate random temporary filename + unsigned short randv = 0; + GetRandBytes((unsigned char*)&randv, sizeof(randv)); + std::string tmpfn = strprintf("zindex.dat.%04x", randv); + + // serialize zstats, checksum data up to that point, then append checksum + CDataStream ssZstats(SER_DISK, CLIENT_VERSION); + ssZstats << FLATDATA(Params().MessageStart()); + ssZstats << zstats; + uint256 hash = Hash(ssZstats.begin(), ssZstats.end()); + ssZstats << hash; + + // open temp output file, and associate with CAutoFile + boost::filesystem::path pathTmp = GetDataDir() / tmpfn; + FILE *file = fopen(pathTmp.string().c_str(), "wb"); + CAutoFile fileout(file, SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("%s: Failed to open file %s", __func__, pathTmp.string()); + + // Write and commit header, data + try { + fileout << ssZstats; + } catch (const std::exception& e) { + return error("%s: Serialize or I/O error - %s", __func__, e.what()); + } + FileCommit(fileout.Get()); + fileout.fclose(); + + // replace existing zindex.dat, if any, with new zindex.dat.XXXX + if (!RenameOver(pathTmp, pathAddr)) + return error("%s: Rename-into-place failed", __func__); + + return true; +} + bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { LOCK(cs_main); CNodeState *state = State(nodeid); @@ -4553,6 +4650,39 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl if (fZdebug) { //fprintf(stderr,"%s: setting blockchain zstats with zspends=%d, zouts=%d\n", __FUNCTION__, nShieldedSpendsInBlock, nShieldedOutputsInBlock ); } + if (pindex->pprev) { + // If chain stats are zero (such as after restart), load data from zindex.dat + if (pindex->pprev->nChainNotarizations == 0) + pindex->pprev->nChainNotarizations = zstats.nChainNotarizations; + if (pindex->pprev->nChainShieldedTx == 0) + pindex->pprev->nChainShieldedTx = zstats.nChainShieldedTx; + if (pindex->pprev->nChainShieldedOutputs == 0) + pindex->pprev->nChainShieldedOutputs = zstats.nChainShieldedOutputs; + if (pindex->pprev->nChainShieldedSpends == 0) { + pindex->pprev->nChainShieldedSpends = zstats.nChainShieldedSpends; + // TODO: if zstats.nHeight != chainActive.Height() the stats will be off + fprintf(stderr, "%s: loaded anonymity set of %li at stats height=%li vs local height=%d from disk\n", __func__, zstats.nChainShieldedOutputs - zstats.nChainShieldedSpends, zstats.nHeight, chainActive.Height() ); + } + if (pindex->pprev->nChainFullyShieldedTx == 0) + pindex->pprev->nChainFullyShieldedTx = zstats.nChainFullyShieldedTx; + if (pindex->pprev->nChainShieldingTx == 0) + pindex->pprev->nChainShieldingTx = zstats.nChainShieldingTx; + if (pindex->pprev->nChainDeshieldingTx == 0) + pindex->pprev->nChainDeshieldingTx = zstats.nChainDeshieldingTx; + if (pindex->pprev->nChainPayments == 0) { + fprintf(stderr, "%s: setting nChainPayments=%li at height %d\n", __func__, zstats.nChainPayments, chainActive.Height() ); + pindex->pprev->nChainPayments = zstats.nChainPayments; + } + if (pindex->pprev->nChainShieldedPayments == 0) + pindex->pprev->nChainShieldedPayments = zstats.nChainShieldedPayments; + if (pindex->pprev->nChainFullyShieldedPayments == 0) + pindex->pprev->nChainFullyShieldedPayments = zstats.nChainFullyShieldedPayments; + if (pindex->pprev->nChainShieldingPayments == 0) + pindex->pprev->nChainShieldingPayments = zstats.nChainShieldingPayments; + if (pindex->pprev->nChainDeshieldingPayments == 0) + pindex->pprev->nChainDeshieldingPayments = zstats.nChainDeshieldingPayments; + } + pindex->nChainNotarizations = (pindex->pprev ? pindex->pprev->nChainNotarizations : 0) + pindex->nNotarizations; pindex->nChainShieldedTx = (pindex->pprev ? pindex->pprev->nChainShieldedTx : 0) + pindex->nShieldedTx; pindex->nChainShieldedOutputs = (pindex->pprev ? pindex->pprev->nChainShieldedOutputs : 0) + pindex->nShieldedOutputs; @@ -4565,6 +4695,23 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindex->nChainFullyShieldedPayments = (pindex->pprev ? pindex->pprev->nChainFullyShieldedPayments : 0) + pindex->nFullyShieldedPayments; pindex->nChainShieldingPayments = (pindex->pprev ? pindex->pprev->nChainShieldingPayments : 0) + pindex->nShieldingPayments; pindex->nChainDeshieldingPayments = (pindex->pprev ? pindex->pprev->nChainDeshieldingPayments : 0) + pindex->nDeshieldingPayments; + + // Update in-memory structure that gets serialized to zindex.dat + zstats.nHeight = pindex->GetHeight(); + zstats.nChainNotarizations = pindex->nChainNotarizations ; + zstats.nChainShieldedTx = pindex->nChainShieldedTx ; + zstats.nChainShieldedOutputs = pindex->nChainShieldedOutputs ; + zstats.nChainShieldedSpends = pindex->nChainShieldedSpends ; + zstats.nChainFullyShieldedTx = pindex->nChainFullyShieldedTx ; + zstats.nChainShieldingTx = pindex->nChainShieldingTx ; + zstats.nChainDeshieldingTx = pindex->nChainDeshieldingTx ; + zstats.nChainPayments = pindex->nChainPayments ; + zstats.nChainShieldedPayments = pindex->nChainShieldedPayments ; + zstats.nChainFullyShieldedPayments = pindex->nChainFullyShieldedPayments ; + zstats.nChainShieldingPayments = pindex->nChainShieldingPayments ; + zstats.nChainDeshieldingPayments = pindex->nChainDeshieldingPayments ; + fprintf(stderr,"%s: setting zstats with height,zouts,zspends,anonset=%li,%li,%li,%li\n", __FUNCTION__, zstats.nHeight, zstats.nChainShieldedOutputs, zstats.nChainShieldedSpends, zstats.nChainShieldedOutputs - zstats.nChainShieldedSpends); + } if (pindex->pprev) { @@ -4603,6 +4750,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl } } + if (fZindex) fprintf(stderr, "ht.%d, ShieldedPayments=%d, ShieldedTx=%d, ShieldedOutputs=%d, FullyShieldedTx=%d, ntz=%d\n", pindexNew->GetHeight(), nShieldedPayments, nShieldedTx, nShieldedOutputs, nFullyShieldedTx, nNotarizations ); diff --git a/src/main.h b/src/main.h index 078e48cef..676a069cd 100644 --- a/src/main.h +++ b/src/main.h @@ -950,4 +950,114 @@ std::pair>, uint64_t> DrainRecent void SetChainNotifiedSequence(uint64_t recentlyConflictedSequence); bool ChainIsFullyNotified(); +class CZindexStats +{ +//private: +public: + int64_t nHeight; + int64_t nChainTx; + int64_t nChainNotarizations; + int64_t nChainPayments; + int64_t nChainShieldedTx; + int64_t nChainShieldedOutputs; + int64_t nChainShieldedSpends; + int64_t nChainFullyShieldedTx; + int64_t nChainShieldingPayments; + int64_t nChainShieldedPayments; + int64_t nChainFullyShieldedPayments; + int64_t nChainDeshieldingTx; + int64_t nChainDeshieldingPayments; + int64_t nChainShieldingTx; + + size_t Height() const + { + return nHeight; + } + + void Clear() + { + LOCK(cs_main); + nChainTx=0; + nChainNotarizations=0; + nChainPayments=0; + nChainShieldedTx=0; + nChainShieldedOutputs=0; + nChainShieldedSpends=0; + nChainFullyShieldedTx=0; + nChainShieldingPayments=0; + nChainShieldedPayments=0; + nChainFullyShieldedPayments=0; + nChainDeshieldingTx=0; + nChainDeshieldingPayments=0; + nChainShieldingTx=0; + } + + CZindexStats() + { + Clear(); + } + + ~CZindexStats() + { + } + + template void Serialize(Stream &s) const + { + LOCK(cs_main); + + // So we can detect a new version and force a rescan + unsigned char nVersion = 1; + s << nVersion; + s << nHeight; + s << nChainTx; + s << nChainNotarizations; + s << nChainPayments; + s << nChainShieldedTx; + s << nChainShieldedOutputs; + s << nChainShieldedSpends; + s << nChainFullyShieldedTx; + s << nChainShieldingPayments; + s << nChainShieldedPayments; + s << nChainFullyShieldedPayments; + s << nChainDeshieldingTx; + s << nChainDeshieldingPayments; + s << nChainShieldingTx; + } + + template void Unserialize(Stream& s) + { + LOCK(cs_main); + + Clear(); + unsigned char nVersion; + s >> nVersion; + s >> nHeight; + s >> nChainTx; + s >> nChainNotarizations; + s >> nChainPayments; + s >> nChainShieldedTx; + s >> nChainShieldedOutputs; + s >> nChainShieldedSpends; + s >> nChainFullyShieldedTx; + s >> nChainShieldingPayments; + s >> nChainShieldedPayments; + s >> nChainFullyShieldedPayments; + s >> nChainDeshieldingTx; + s >> nChainDeshieldingPayments; + s >> nChainShieldingTx; + } +}; + +// Wrapper for zindex.dat stats +class CZindexDB +{ +private: + boost::filesystem::path pathAddr; +public: + CZindexDB(); + bool Write(const CZindexStats& zstats); + bool Read(CZindexStats& zstats); +}; + + #endif // HUSH_MAIN_H diff --git a/src/net.cpp b/src/net.cpp index 808df788c..218f766d3 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -48,6 +48,9 @@ using namespace hush; // Satoshi originally used 10 seconds(!), did they know something Peter Wuille didn't? #define DUMP_ADDRESSES_INTERVAL 300 +// This is every 2 blocks, on avg, on HUSH3 +#define DUMP_ZINDEX_INTERVAL 150 + #if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 #endif @@ -92,7 +95,6 @@ namespace { // Global state variables extern uint16_t ASSETCHAINS_P2PPORT; extern char SMART_CHAIN_SYMBOL[65]; - bool fDiscover = true; bool fListen = true; uint64_t nLocalServices = NODE_NETWORK | NODE_NSPV; @@ -103,6 +105,7 @@ static CNode* pnodeLocalHost = NULL; uint64_t nLocalHostNonce = 0; static std::vector vhListenSocket; CAddrMan addrman; +CZindexStats zstats; int nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; bool fAddressesInitialized = false; std::string strSubVersion; @@ -1400,6 +1403,16 @@ void DumpAddresses() LogPrint("net", "Flushed %d addresses to peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); } +void DumpZindexStats() +{ + int64_t nStart = GetTimeMillis(); + + CZindexDB zdb; + zdb.Write(zstats); + + LogPrintf("Flushed stats at height %li to zindex.dat %dms\n", zstats.Height(), GetTimeMillis() - nStart); +} + void static ProcessOneShot() { string strDest; @@ -1909,8 +1922,37 @@ void static Discover(boost::thread_group& threadGroup) #endif } +//extern CWallet pwalletMain; void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) { + if (fZindex) { + uiInterface.InitMessage(_("Loading zindex stats...")); + int64_t nStart = GetTimeMillis(); + { + CZindexDB zdb; + if (!zdb.Read(zstats)) { + // The first time nodes use zindex.dat code, no file will be found + // TODO: rescan if invalid only + LogPrintf("Invalid or missing zindex.dat! Generating new...\n"); + + //bool update = true; + //pwalletMain->ScanForWalletTransactions(chainActive.Genesis(),update); + + // We assume this is the first startup with zindex.dat code, and serialize current data to disk. + DumpZindexStats(); + + // Now read-in the stats we just wrote to disk to memory + if(!zdb.Read(zstats)) { + LogPrintf("Invalid or missing zindex.dat! Stats may be corrupt\n"); + } else { + LogPrintf("Loaded stats at height %li from zindex.dat %dms\n", zstats.Height(), GetTimeMillis() - nStart); + } + } else { + LogPrintf("Loaded stats at height %li from zindex.dat %dms\n", zstats.Height(), GetTimeMillis() - nStart); + } + } + } + uiInterface.InitMessage(_("Loading addresses...")); // Load addresses for peers.dat int64_t nStart = GetTimeMillis(); @@ -1919,8 +1961,7 @@ void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) if (!adb.Read(addrman)) LogPrintf("Invalid or missing peers.dat! This can happen when upgrading. Whatevz, recreating\n"); } - LogPrintf("Loaded %i addresses from peers.dat %dms\n", - addrman.size(), GetTimeMillis() - nStart); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); fAddressesInitialized = true; if (semOutbound == NULL) { @@ -1968,6 +2009,11 @@ void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) // Dump network addresses scheduler.scheduleEvery(&DumpAddresses, DUMP_ADDRESSES_INTERVAL); + + // Dump zindex stats if -zindex is enabled + if (fZindex) { + scheduler.scheduleEvery(&DumpZindexStats, DUMP_ZINDEX_INTERVAL); + } } bool StopNode() @@ -1977,6 +2023,9 @@ bool StopNode() for (int i=0; i<(MAX_OUTBOUND_CONNECTIONS + MAX_FEELER_CONNECTIONS); i++) semOutbound->post(); + // persist current zindex stats to disk before we exit + DumpZindexStats(); + if (HUSH_NSPV_FULLNODE && fAddressesInitialized) { DumpAddresses(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7c38ef1eb..609f26420 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1660,10 +1660,12 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 0 and the block's height - 1"); } } + LogPrintf("%s: blockcount = %d\n", __func__, blockcount); const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->GetHeight() - blockcount); int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + LogPrintf("%s: pindexPast.height = %d, pindex.height = %d\n", __func__, pindexPast->GetHeight(), pindex->GetHeight() ); UniValue ret(UniValue::VOBJ); ret.pushKV("time", (int64_t)pindex->nTime); @@ -1695,6 +1697,7 @@ UniValue getchaintxstats(const UniValue& params, bool fHelp, const CPubKey& mypk ret.pushKV("window_tx_count", nTxDiff); ret.pushKV("window_interval", nTimeDiff); int64_t nPaymentsDiff = pindex->nChainPayments - pindexPast->nChainPayments; + LogPrintf("%s: pindexPast.nChainPayments = %d, pindex.nChainPayments = %d\n", __func__, pindexPast->nChainPayments, pindex->nChainPayments ); int64_t nShieldedTxDiff = pindex->nChainShieldedTx - pindexPast->nChainShieldedTx; int64_t nShieldingTxDiff = pindex->nChainShieldingTx - pindexPast->nChainShieldingTx; int64_t nDeshieldingTxDiff = pindex->nChainDeshieldingTx - pindexPast->nChainDeshieldingTx;