From e8dc755f068ab82753c2dc05d1f376cb7ed380ed Mon Sep 17 00:00:00 2001 From: Duke Date: Thu, 13 Apr 2023 23:30:23 -0400 Subject: [PATCH 01/17] Reduce memory usage of CBlockIndex Ported code from https://github.com/zcash/zcash/pull/6192 with various changes needed for the Hush codebase. --- src/chain.cpp | 45 +++++++++++++++++++++++ src/chain.h | 79 ++++++++++++++++++++++++++-------------- src/hush_nSPV_fullnode.h | 14 ++++++- src/main.cpp | 14 ++++++- src/main.h | 1 + src/rest.cpp | 14 +++++-- src/rpc/blockchain.cpp | 21 ++++++----- src/txdb.cpp | 47 ++++++++++++++++++++---- src/txdb.h | 4 +- 9 files changed, 187 insertions(+), 52 deletions(-) diff --git a/src/chain.cpp b/src/chain.cpp index 1e87b22e1..dfa993f38 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -19,12 +19,57 @@ ******************************************************************************/ #include "chain.h" +#include "main.h" +#include "txdb.h" using namespace std; /** * CChain implementation */ +void CBlockIndex::TrimSolution() +{ + AssertLockHeld(cs_main); + + // We can correctly trim a solution as soon as the block index entry has been added + // to leveldb. Updates to the block index entry (to update validity status) will be + // handled by re-reading the solution from the existing db entry. It does not help to + // try to avoid these reads by gating trimming on the validity status: the re-reads are + // efficient anyway because of caching in leveldb, and most of them are unavoidable. + if (HasSolution()) { + std::vector empty; + nSolution.swap(empty); + } +} + +CBlockHeader CBlockIndex::GetBlockHeader() const +{ + AssertLockHeld(cs_main); + + CBlockHeader header; + header.nVersion = nVersion; + if (pprev) { + header.hashPrevBlock = pprev->GetBlockHash(); + } + header.hashMerkleRoot = hashMerkleRoot; + // Hush does not have this, maybe some day + // header.hashBlockCommitments = hashBlockCommitments; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + if (HasSolution()) { + header.nSolution = nSolution; + } else { + CDiskBlockIndex dbindex; + if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw std::runtime_error("Failed to read index entry"); + } + header.nSolution = dbindex.GetSolution(); + } + return header; +} + void CChain::SetTip(CBlockIndex *pindex) { lastTip = pindex; if (pindex == NULL) { diff --git a/src/chain.h b/src/chain.h index eb94559bd..03d3b3be2 100644 --- a/src/chain.h +++ b/src/chain.h @@ -27,6 +27,7 @@ class CChainPower; #include "pow.h" #include "tinyformat.h" #include "uint256.h" +#include "util/strencodings.h" #include #include @@ -384,8 +385,14 @@ public: unsigned int nTime; unsigned int nBits; uint256 nNonce; +protected: + // The Equihash solution, if it is stored. Once we know that the block index + // entry is present in leveldb, this field can be cleared via the TrimSolution + // method to save memory. std::vector nSolution; +public: + //! (memory only) Sequential id assigned to distinguish order in which blocks are received. uint32_t nSequenceId; @@ -497,23 +504,15 @@ public: return ret; } - CBlockHeader GetBlockHeader() const - { - CBlockHeader block; - block.nVersion = nVersion; - if (pprev) - block.hashPrevBlock = pprev->GetBlockHash(); - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block; - } + //! Get the block header for this block index. Requires cs_main. + CBlockHeader GetBlockHeader() const; + + //! Clear the Equihash solution to save memory. Requires cs_main. + void TrimSolution(); uint256 GetBlockHash() const { + assert(phashBlock); return *phashBlock; } @@ -540,10 +539,11 @@ public: std::string ToString() const { - return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s)", + return strprintf("CBlockIndex(pprev=%p, nHeight=%d, merkle=%s, hashBlock=%s, HasSolution=%s)", pprev, this->chainPower.nHeight, hashMerkleRoot.ToString(), - GetBlockHash().ToString()); + phashBlock ? GetBlockHash().ToString() : "(nil)", + HasSolution()); } //! Check whether this block index entry is valid up to the passed validity level. @@ -555,6 +555,12 @@ public: return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! Is the Equihash solution stored? + bool HasSolution() const + { + return !nSolution.empty(); + } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) @@ -588,8 +594,11 @@ public: hashPrev = uint256(); } - explicit CDiskBlockIndex(const CBlockIndex* pindex) : CBlockIndex(*pindex) { + explicit CDiskBlockIndex(const CBlockIndex* pindex, std::function()> getSolution) : CBlockIndex(*pindex) { hashPrev = (pprev ? pprev->GetBlockHash() : uint256()); + if (!HasSolution()) { + nSolution = getSolution(); + } } ADD_SERIALIZE_METHODS; @@ -668,20 +677,27 @@ public: } } - uint256 GetBlockHash() const + //! Get the block header for this block index. + CBlockHeader GetBlockHeader() const { - CBlockHeader block; - block.nVersion = nVersion; - block.hashPrevBlock = hashPrev; - block.hashMerkleRoot = hashMerkleRoot; - block.hashFinalSaplingRoot = hashFinalSaplingRoot; - block.nTime = nTime; - block.nBits = nBits; - block.nNonce = nNonce; - block.nSolution = nSolution; - return block.GetHash(); + CBlockHeader header; + header.nVersion = nVersion; + header.hashPrevBlock = hashPrev; + header.hashMerkleRoot = hashMerkleRoot; + // Hush does not have this, maybe some day + // header.hashBlockCommitments = hashBlockCommitments; + header.nTime = nTime; + header.nBits = nBits; + header.nNonce = nNonce; + header.nSolution = nSolution; + return header; } + std::vector GetSolution() const + { + assert(HasSolution()); + return nSolution; + } std::string ToString() const { @@ -692,6 +708,13 @@ public: hashPrev.ToString()); return str; } + +private: + //! This method should not be called on a CDiskBlockIndex. + void TrimSolution() + { + assert(!"called CDiskBlockIndex::TrimSolution"); + } }; /** An in-memory indexed chain of blocks. */ diff --git a/src/hush_nSPV_fullnode.h b/src/hush_nSPV_fullnode.h index 48ae589d8..5124bed35 100644 --- a/src/hush_nSPV_fullnode.h +++ b/src/hush_nSPV_fullnode.h @@ -124,7 +124,19 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height) hdr->nTime = pindex->nTime; hdr->nBits = pindex->nBits; hdr->nNonce = pindex->nNonce; - memcpy(hdr->nSolution,&pindex->nSolution[0],sizeof(hdr->nSolution)); + auto header = pindex->GetBlockHeader(); + + // if (pindex->HasSolution()) { + // header.nSolution = nSolution; + // } else { + CDiskBlockIndex dbindex; + if (!pblocktree->ReadDiskBlockIndex(pindex->GetBlockHash(), dbindex)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw std::runtime_error("Failed to read index entry"); + } + header.nSolution = dbindex.GetSolution(); + // } + memcpy(hdr->nSolution,&header.nSolution[0],sizeof(hdr->nSolution)); return(sizeof(*hdr)); } return(-1); diff --git a/src/main.cpp b/src/main.cpp index 8226e1d9c..f6d6ca270 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3689,7 +3689,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it])); setDirtyFileInfo.erase(it++); } - std::vector vBlocks; + std::vector vBlocks; vBlocks.reserve(setDirtyBlockIndex.size()); for (set::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { vBlocks.push_back(*it); @@ -3698,6 +3698,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode(state, "Files to write to block index database"); } + // Now that we have written the block indices to the database, we do not + // need to store solutions for these CBlockIndex objects in memory. + // cs_main must be held here. + for (CBlockIndex *pblockindex : vBlocks) { + pblockindex->TrimSolution(); + } } // Finally remove any pruned files if (fFlushForPrune) @@ -6579,7 +6585,11 @@ void static CheckBlockIndex() } } } - // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // try { + // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // } catch (const runtime_error&) { + // assert(!"Failed to read index entry"); + // } // End: actual consistency checks. // Try descending into the first subnode. diff --git a/src/main.h b/src/main.h index 66357b3d0..cfd9fed19 100644 --- a/src/main.h +++ b/src/main.h @@ -39,6 +39,7 @@ #include "spentindex.h" #include "sync.h" #include "tinyformat.h" +#include "txdb.h" #include "txmempool.h" #include "uint256.h" diff --git a/src/rest.cpp b/src/rest.cpp index 36b4279b1..3536d790a 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -151,6 +151,7 @@ static bool rest_headers(HTTPRequest* req, std::vector headers; headers.reserve(count); + CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); { LOCK(cs_main); BlockMap::const_iterator it = mapBlockIndex.find(hash); @@ -161,11 +162,16 @@ static bool rest_headers(HTTPRequest* req, break; pindex = chainActive.Next(pindex); } - } - CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); - BOOST_FOREACH(const CBlockIndex *pindex, headers) { - ssHeader << pindex->GetBlockHeader(); + if (rf == RF_BINARY || rf == RF_HEX) { + try { + for (const CBlockIndex *pindex : headers) { + ssHeader << pindex->GetBlockHeader(); + } + } catch (const std::runtime_error&) { + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Failed to read index entry"); + } + } } switch (rf) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1c34cd1b0..760310e60 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -147,7 +147,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) result.push_back(Pair("finalsaplingroot", blockindex->hashFinalSaplingRoot.GetHex())); result.push_back(Pair("time", (int64_t)blockindex->nTime)); result.push_back(Pair("nonce", blockindex->nNonce.GetHex())); - result.push_back(Pair("solution", HexStr(blockindex->nSolution))); + result.pushKV("solution", HexStr(blockindex->GetBlockHeader().nSolution)); result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits))); result.push_back(Pair("difficulty", GetDifficulty(blockindex))); result.push_back(Pair("chainwork", blockindex->chainPower.chainWork.GetHex())); @@ -694,15 +694,18 @@ UniValue getblockheader(const UniValue& params, bool fHelp, const CPubKey& mypk) CBlockIndex* pblockindex = mapBlockIndex[hash]; - if (!fVerbose) - { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); - ssBlock << pblockindex->GetBlockHeader(); - std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); - return strHex; + try { + if (!fVerbose) { + CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); + ssBlock << pblockindex->GetBlockHeader(); + std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); + return strHex; + } else { + return blockheaderToJSON(pblockindex); + } + } catch (const runtime_error&) { + throw JSONRPCError(RPC_DATABASE_ERROR, "Failed to read index entry"); } - - return blockheaderToJSON(pblockindex); } UniValue getblock(const UniValue& params, bool fHelp, const CPubKey& mypk) diff --git a/src/txdb.cpp b/src/txdb.cpp index dbba3654f..602987412 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -269,18 +269,33 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { return true; } -bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { +bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { CDBBatch batch(*this); if (fZdebug) fprintf(stderr, "%s: Writing block files\n", __FUNCTION__); - for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); + for (const auto& it : fileInfo) { + batch.Write(make_pair(DB_BLOCK_FILES, it.first), *it.second); } batch.Write(DB_LAST_BLOCK, nLastFile); if (fZdebug) fprintf(stderr, "%s: Writing block index\n", __FUNCTION__); - for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { - batch.Write(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); + for (const auto& it : blockinfo) { + std::pair key = make_pair(DB_BLOCK_INDEX, it->GetBlockHash()); + try { + CDiskBlockIndex dbindex {it, [this, &key]() { + // It can happen that the index entry is written, then the Equihash solution is cleared from memory, + // then the index entry is rewritten. In that case we must read the solution from the old entry. + CDiskBlockIndex dbindex_old; + if (!Read(key, dbindex_old)) { + LogPrintf("%s: Failed to read index entry", __func__); + throw runtime_error("Failed to read index entry"); + } + return dbindex_old.GetSolution(); + }}; + batch.Write(key, dbindex); + } catch (const runtime_error&) { + return false; + } } return WriteBatch(batch, true); } @@ -293,6 +308,11 @@ bool CBlockTreeDB::EraseBatchSync(const std::vector& blockin return WriteBatch(batch, true); } +bool CBlockTreeDB::ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const { + return Read(make_pair(DB_BLOCK_INDEX, blockhash), dbindex); +} + + bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const { return Read(make_pair(DB_TXINDEX, txid), pos); } @@ -692,7 +712,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; - pindexNew->nSolution = diskindex.nSolution; + // the Equihash solution will be loaded lazily from the dbindex entry + // pindexNew->nSolution = diskindex.nSolution; pindexNew->nStatus = diskindex.nStatus; pindexNew->nCachedBranchId = diskindex.nCachedBranchId; pindexNew->nTx = diskindex.nTx; @@ -716,7 +737,19 @@ bool CBlockTreeDB::LoadBlockIndexGuts() //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks - auto header = pindexNew->GetBlockHeader(); + CBlockHeader header; + { + LOCK(cs_main); + try { + header = pindexNew->GetBlockHeader(); + } catch (const runtime_error&) { + return error("LoadBlockIndex(): failed to read index entry: diskindex hash = %s", + diskindex.GetBlockHash().ToString()); + } + } + if (header.GetHash() != diskindex.GetBlockHash()) + return error("LoadBlockIndex(): inconsistent header vs diskindex hash: header hash = %s, diskindex hash = %s", + header.GetHash().ToString(), diskindex.GetBlockHash().ToString()); if (header.GetHash() != pindexNew->GetBlockHash()) return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", diskindex.ToString(), pindexNew->ToString()); diff --git a/src/txdb.h b/src/txdb.h index 0b0ac376c..927929960 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -26,6 +26,7 @@ #include "coins.h" #include "dbwrapper.h" +#include "chain.h" #include #include #include @@ -91,12 +92,13 @@ private: CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); public: - bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo); + bool WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo); bool EraseBatchSync(const std::vector& blockinfo); bool ReadBlockFileInfo(int nFile, CBlockFileInfo &fileinfo) const; bool ReadLastBlockFile(int &nFile) const; bool WriteReindexing(bool fReindex); bool ReadReindexing(bool &fReindex) const; + bool ReadDiskBlockIndex(const uint256 &blockhash, CDiskBlockIndex &dbindex) const; bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) const; bool WriteTxIndex(const std::vector > &list); bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) const; From fab9a7f51d138ec2e2025e7b668d620205ddd070 Mon Sep 17 00:00:00 2001 From: onryo Date: Sun, 16 Apr 2023 15:53:48 +0000 Subject: [PATCH 02/17] Add libleveldb-dev to build dependencies Fixes "leveldb/db.h: No such file or directory" error. --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index c13109788..fa10114c4 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,7 +25,7 @@ sudo swapon /swapfile # install build dependencies sudo apt-get install build-essential pkg-config libc6-dev m4 g++-multilib \ autoconf libtool ncurses-dev unzip git zlib1g-dev wget \ - bsdmainutils automake curl unzip nano libsodium-dev cmake + bsdmainutils automake curl unzip nano libsodium-dev libleveldb-dev cmake # clone git repo git clone https://git.hush.is/hush/hush3 cd hush3 From 4d6c4513146df7ab9d86b419777f0bdaf95634d5 Mon Sep 17 00:00:00 2001 From: Duke Date: Mon, 17 Apr 2023 08:38:30 -0700 Subject: [PATCH 03/17] Try to fix #290 --- src/dbwrapper.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 13c6a0e96..d16be9307 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -12,8 +12,8 @@ #include "util.h" #include "version.h" #include -#include -#include +#include "leveldb/db.h" +#include "leveldb/write_batch.h" static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; From d181029d2733716da5d2f91a3a92ab4b55892371 Mon Sep 17 00:00:00 2001 From: Duke Date: Mon, 17 Apr 2023 08:46:44 -0700 Subject: [PATCH 04/17] Include path for leveldb when compiling cc #290 --- src/cc/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cc/Makefile b/src/cc/Makefile index 24b94226a..54f290573 100644 --- a/src/cc/Makefile +++ b/src/cc/Makefile @@ -3,9 +3,9 @@ CC = gcc CC_DARWIN = g++-6 CC_WIN = x86_64-w64-mingw32-gcc-posix CFLAGS = -arch x86_64 -CXXFLAGS_DARWIN = -std=c++11 -arch x86_64 -I/usr/local/Cellar/gcc\@6/6.4.0_2/include/c++/6.4.0/ -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -c -Wl,-undefined -Wl,dynamic_lookup -dynamiclib -CXXFLAGS = -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared -c -CXXFLAGS_WIN = -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared -c +CXXFLAGS_DARWIN = -std=c++11 -arch x86_64 -I/usr/local/Cellar/gcc\@6/6.4.0_2/include/c++/6.4.0/ -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I -I../leveldb/include -I.. -I. -fPIC -c -Wl,-undefined -Wl,dynamic_lookup -dynamiclib +CXXFLAGS = -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared -c +CXXFLAGS_WIN = -std=c++11 -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared -c DEBUGFLAGS = -O0 -D _DEBUG RELEASEFLAGS = -O2 -D NDEBUG -combine -fwhole-program $(info $(OS)) From bc1d6be55f3f53079702aa7ed6319406378f0705 Mon Sep 17 00:00:00 2001 From: Duke Date: Mon, 17 Apr 2023 14:32:02 -0700 Subject: [PATCH 05/17] Add leveldb/include header dir to src/Makefile.am --- src/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile.am b/src/Makefile.am index e5e2ca248..6178fa19e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -39,6 +39,7 @@ BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/include BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/src BITCOIN_INCLUDES += -I$(srcdir)/cryptoconditions/src/asn BITCOIN_INCLUDES += -I$(srcdir)/univalue/include +BITCOIN_INCLUDES += -I$(srcdir)/leveldb/include if TARGET_WINDOWS LIBBITCOIN_SERVER=libbitcoin_server.a -lcurl From f8939d2fd9affeea0350c97b0788f1770d2f933e Mon Sep 17 00:00:00 2001 From: Duke Date: Wed, 19 Apr 2023 10:38:21 -0400 Subject: [PATCH 06/17] Fix src/cc compile --- src/cc/Makefile_custom | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cc/Makefile_custom b/src/cc/Makefile_custom index 3989eefa3..5cedceec0 100644 --- a/src/cc/Makefile_custom +++ b/src/cc/Makefile_custom @@ -3,9 +3,9 @@ CC = gcc CC_DARWIN = g++-8 CC_WIN = x86_64-w64-mingw32-gcc-posix CC_AARCH64 = aarch64-linux-gnu-g++ -CFLAGS_DARWIN = -DBUILD_CUSTOMCC -std=c++11 -arch x86_64 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -Wl,-undefined -Wl,dynamic_lookup -Wno-write-strings -shared -dynamiclib -CFLAGS = -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared -CFLAGS_WIN = -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/x86_64-w64-mingw32/include -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I.. -I. -fPIC -shared +CFLAGS_DARWIN = -DBUILD_CUSTOMCC -std=c++11 -arch x86_64 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -Wl,-undefined -Wl,dynamic_lookup -Wno-write-strings -shared -dynamiclib +CFLAGS = -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/$(shell echo `../..//depends/config.guess`/include) -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared +CFLAGS_WIN = -Wno-write-strings -DBUILD_CUSTOMCC -std=c++11 -I../secp256k1/include -I../../depends/x86_64-w64-mingw32/include -I../univalue/include -I../cryptoconditions/include -I../cryptoconditions/src -I../cryptoconditions/src/asn -I../leveldb/include -I.. -I. -fPIC -shared DEBUGFLAGS = -O0 -D _DEBUG RELEASEFLAGS = -O2 -D NDEBUG -combine -fwhole-program $(info $(OS)) From 513ebc5bb4c6a1edb5e199aa6f1aa5fe4947f04b Mon Sep 17 00:00:00 2001 From: Duke Date: Wed, 19 Apr 2023 11:01:20 -0400 Subject: [PATCH 07/17] Say which height we are at when we get an assertion from GetBlockHash --- src/chain.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/chain.h b/src/chain.h index 03d3b3be2..46136df99 100644 --- a/src/chain.h +++ b/src/chain.h @@ -512,7 +512,10 @@ public: uint256 GetBlockHash() const { - assert(phashBlock); + if(phashBlock == NULL) { + fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d\n", this->GetHeight()); + assert(phashBlock); + } return *phashBlock; } From cf1274a42a7e2416d4958f91238edcb9cfcd3817 Mon Sep 17 00:00:00 2001 From: Duke Date: Thu, 20 Apr 2023 04:47:20 -0700 Subject: [PATCH 08/17] Remove unnecessary consistency checks As per https://github.com/zcash/zcash/issues/6532 this should speed up startup times by roughly 3X and drastically reduce disk i/o on startup. --- src/txdb.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/txdb.cpp b/src/txdb.cpp index 602987412..22b70f5d5 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -737,6 +737,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() //fprintf(stderr,"loadguts ht.%d\n",pindexNew->GetHeight()); // Consistency checks + /* CBlockHeader header; { LOCK(cs_main); @@ -753,6 +754,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() if (header.GetHash() != pindexNew->GetBlockHash()) return error("LoadBlockIndex(): block header inconsistency detected: on-disk = %s, in-memory = %s", diskindex.ToString(), pindexNew->ToString()); + if ( 0 ) // POW will be checked before any block is connected { uint8_t pubkey33[33]; @@ -760,6 +762,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts() if (!CheckProofOfWork(header,pubkey33,pindexNew->GetHeight(),Params().GetConsensus())) return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString()); } + */ pcursor->Next(); } else { return error("LoadBlockIndex() : failed to read value"); From ca6a1c48bb74e0487cef8c170b3d59170ff70abb Mon Sep 17 00:00:00 2001 From: Duke Date: Sun, 23 Apr 2023 06:02:22 -0700 Subject: [PATCH 09/17] Make GetBlockHash() calculate the blockhash if phashBlock is null --- src/chain.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/chain.h b/src/chain.h index 46136df99..618411493 100644 --- a/src/chain.h +++ b/src/chain.h @@ -513,8 +513,11 @@ public: uint256 GetBlockHash() const { if(phashBlock == NULL) { - fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d\n", this->GetHeight()); - assert(phashBlock); + auto header = this->GetBlockHeader(); + auto blockhash = header.GetHash(); + fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash->ToString() ); + // assert(phashBlock); + return blockhash; } return *phashBlock; } From b197c18ffc45aa1d00fa554a73d72085cc6eb00d Mon Sep 17 00:00:00 2001 From: Duke Date: Sun, 23 Apr 2023 09:05:51 -0400 Subject: [PATCH 10/17] . --- src/chain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain.h b/src/chain.h index 618411493..b72fe9cab 100644 --- a/src/chain.h +++ b/src/chain.h @@ -515,7 +515,7 @@ public: if(phashBlock == NULL) { auto header = this->GetBlockHeader(); auto blockhash = header.GetHash(); - fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash->ToString() ); + fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash.ToString() ); // assert(phashBlock); return blockhash; } From 9602e715ed4ca49a57353fd6b73af749ce618aca Mon Sep 17 00:00:00 2001 From: Duke Date: Sun, 23 Apr 2023 09:13:21 -0400 Subject: [PATCH 11/17] Fix compile issue --- src/chain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain.h b/src/chain.h index b72fe9cab..67c63cf09 100644 --- a/src/chain.h +++ b/src/chain.h @@ -515,7 +515,7 @@ public: if(phashBlock == NULL) { auto header = this->GetBlockHeader(); auto blockhash = header.GetHash(); - fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash.ToString() ); + fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash.ToString().c_str() ); // assert(phashBlock); return blockhash; } From 883fbe407f6f7ed19e1420201c60d768188eba15 Mon Sep 17 00:00:00 2001 From: Duke Date: Sun, 23 Apr 2023 06:23:18 -0700 Subject: [PATCH 12/17] Revert "Add libleveldb-dev to build dependencies" This reverts commit fab9a7f51d138ec2e2025e7b668d620205ddd070. --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index fa10114c4..c13109788 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -25,7 +25,7 @@ sudo swapon /swapfile # install build dependencies sudo apt-get install build-essential pkg-config libc6-dev m4 g++-multilib \ autoconf libtool ncurses-dev unzip git zlib1g-dev wget \ - bsdmainutils automake curl unzip nano libsodium-dev libleveldb-dev cmake + bsdmainutils automake curl unzip nano libsodium-dev cmake # clone git repo git clone https://git.hush.is/hush/hush3 cd hush3 From e993acb2f04f5f9096da3b19e9861d4adb357c11 Mon Sep 17 00:00:00 2001 From: Duke Date: Tue, 13 Jun 2023 16:21:25 -0400 Subject: [PATCH 13/17] Fix missing hashFinalSaplingRoot in block header --- src/chain.cpp | 5 ++--- src/chain.h | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/chain.cpp b/src/chain.cpp index dfa993f38..50ccc6472 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -52,8 +52,7 @@ CBlockHeader CBlockIndex::GetBlockHeader() const header.hashPrevBlock = pprev->GetBlockHash(); } header.hashMerkleRoot = hashMerkleRoot; - // Hush does not have this, maybe some day - // header.hashBlockCommitments = hashBlockCommitments; + header.hashFinalSaplingRoot = hashFinalsaplingRoot; header.nTime = nTime; header.nBits = nBits; header.nNonce = nNonce; @@ -62,7 +61,7 @@ CBlockHeader CBlockIndex::GetBlockHeader() const } else { CDiskBlockIndex dbindex; if (!pblocktree->ReadDiskBlockIndex(GetBlockHash(), dbindex)) { - LogPrintf("%s: Failed to read index entry", __func__); + LogPrintf("%s: ReadDiskBlockIndex failed to read index entry of block %s", __func__, GetBlockHash().ToString().c_str()); throw std::runtime_error("Failed to read index entry"); } header.nSolution = dbindex.GetSolution(); diff --git a/src/chain.h b/src/chain.h index 67c63cf09..ce9bc1602 100644 --- a/src/chain.h +++ b/src/chain.h @@ -690,8 +690,7 @@ public: header.nVersion = nVersion; header.hashPrevBlock = hashPrev; header.hashMerkleRoot = hashMerkleRoot; - // Hush does not have this, maybe some day - // header.hashBlockCommitments = hashBlockCommitments; + header.hashFinalSaplingRoot = hashFinalSaplingRoot; header.nTime = nTime; header.nBits = nBits; header.nNonce = nNonce; From 31d97c7c9ce488be427e8cf651f46b01048251ce Mon Sep 17 00:00:00 2001 From: Duke Date: Tue, 13 Jun 2023 16:24:54 -0400 Subject: [PATCH 14/17] Spell it correctly --- src/chain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain.cpp b/src/chain.cpp index 50ccc6472..0b5ac609d 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -52,7 +52,7 @@ CBlockHeader CBlockIndex::GetBlockHeader() const header.hashPrevBlock = pprev->GetBlockHash(); } header.hashMerkleRoot = hashMerkleRoot; - header.hashFinalSaplingRoot = hashFinalsaplingRoot; + header.hashFinalSaplingRoot = hashFinalSaplingRoot; header.nTime = nTime; header.nBits = nBits; header.nNonce = nNonce; From 13612bb1c89388e8142dad69a68bd9ad35849b2f Mon Sep 17 00:00:00 2001 From: Duke Date: Tue, 13 Jun 2023 22:58:36 -0400 Subject: [PATCH 15/17] Better error message when hashPrevBlock != view.GetBestBlock --- src/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index f6d6ca270..786de3397 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3243,7 +3243,8 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin uint256 hashPrevBlock = pindex->pprev == NULL ? uint256() : pindex->pprev->GetBlockHash(); if ( hashPrevBlock != view.GetBestBlock() ) { - fprintf(stderr,"ConnectBlock(): hashPrevBlock != view.GetBestBlock()\n"); + fprintf(stderr,"ConnectBlock(): hashPrevBlock != view.GetBestBlock() %s != %s\n", hashPrevBlock.ToString().c_str(), view.GetBestBlock().ToString().c_str() ); + return state.DoS(1, error("ConnectBlock(): hashPrevBlock != view.GetBestBlock()"), REJECT_INVALID, "hashPrevBlock-not-bestblock"); } From f05cb0cfbd36ac0050bfe045ba8f26f521b14983 Mon Sep 17 00:00:00 2001 From: Duke Date: Wed, 14 Jun 2023 00:08:16 -0400 Subject: [PATCH 16/17] Fix nspv --- src/hush_nSPV_fullnode.h | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/hush_nSPV_fullnode.h b/src/hush_nSPV_fullnode.h index 5124bed35..d93deb686 100644 --- a/src/hush_nSPV_fullnode.h +++ b/src/hush_nSPV_fullnode.h @@ -124,19 +124,8 @@ int32_t NSPV_setequihdr(struct NSPV_equihdr *hdr,int32_t height) hdr->nTime = pindex->nTime; hdr->nBits = pindex->nBits; hdr->nNonce = pindex->nNonce; - auto header = pindex->GetBlockHeader(); - // if (pindex->HasSolution()) { - // header.nSolution = nSolution; - // } else { - CDiskBlockIndex dbindex; - if (!pblocktree->ReadDiskBlockIndex(pindex->GetBlockHash(), dbindex)) { - LogPrintf("%s: Failed to read index entry", __func__); - throw std::runtime_error("Failed to read index entry"); - } - header.nSolution = dbindex.GetSolution(); - // } - memcpy(hdr->nSolution,&header.nSolution[0],sizeof(hdr->nSolution)); + memcpy(hdr->nSolution,&pindex->GetBlockHeader().nSolution[0],sizeof(hdr->nSolution)); return(sizeof(*hdr)); } return(-1); From d99ab44bfb0075cdec028f7d208d5a382bef2333 Mon Sep 17 00:00:00 2001 From: Duke Date: Mon, 23 Oct 2023 22:41:29 -0400 Subject: [PATCH 17/17] Fix implementation of GetBlockHash in CBlockIndex and CDiskBlockIndex --- src/chain.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/chain.h b/src/chain.h index ce9bc1602..27ee5adcc 100644 --- a/src/chain.h +++ b/src/chain.h @@ -512,13 +512,7 @@ public: uint256 GetBlockHash() const { - if(phashBlock == NULL) { - auto header = this->GetBlockHeader(); - auto blockhash = header.GetHash(); - fprintf(stderr, "GetBlockHash: phashBlock NULL at height %d, returning blockhash=%s instead\n", this->GetHeight(), blockhash.ToString().c_str() ); - // assert(phashBlock); - return blockhash; - } + assert(phashBlock); return *phashBlock; } @@ -683,6 +677,11 @@ public: } } + uint256 GetBlockHash() const + { + return GetBlockHeader().GetHash(); + } + //! Get the block header for this block index. CBlockHeader GetBlockHeader() const {