diff --git a/depends/packages/libzerocash.mk b/depends/packages/libzerocash.mk index 0d7ee781c..2a1c2df2c 100644 --- a/depends/packages/libzerocash.mk +++ b/depends/packages/libzerocash.mk @@ -2,8 +2,8 @@ package=libzerocash $(package)_download_path=https://github.com/Electric-Coin-Company/$(package)/archive/ $(package)_file_name=$(package)-$($(package)_git_commit).tar.gz $(package)_download_file=$($(package)_git_commit).tar.gz -$(package)_sha256_hash=ef9cd53db6eedea3a5d24551d16d9f23dd52277e91296a14539faa027770ad23 -$(package)_git_commit=e79cd2dfee8213d49b6c2a8b2353a38d7563c965 +$(package)_sha256_hash=abd2c449a8f9b54668e6cc6ed69b93cd6b5a89de5441c0b544c56526b36d1445 +$(package)_git_commit=53c30e67808d08bf46775ca0fa2d2e20eac356b0 $(package)_dependencies=libsnark crypto++ openssl boost libgmp $(package)_patches= diff --git a/src/coins.cpp b/src/coins.cpp index a41d5a310..a48c66f3f 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -40,20 +40,30 @@ bool CCoins::Spend(uint32_t nPos) Cleanup(); return true; } - +bool CCoinsView::GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const { return false; } bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } -bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } +uint256 CCoinsView::GetBestAnchor() const { return uint256(); }; +bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors) { return false; } bool CCoinsView::GetStats(CCoinsStats &stats) const { return false; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } + +bool CCoinsViewBacked::GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const { return base->GetAnchorAt(rt, tree); } bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } +uint256 CCoinsViewBacked::GetBestAnchor() const { return base->GetBestAnchor(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } -bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } +bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors) { return base->BatchWrite(mapCoins, hashBlock, hashAnchor, mapAnchors); } bool CCoinsViewBacked::GetStats(CCoinsStats &stats) const { return base->GetStats(stats); } CCoinsKeyHasher::CCoinsKeyHasher() : salt(GetRandHash()) {} @@ -87,6 +97,70 @@ CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const return ret; } + +bool CCoinsViewCache::GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const { + CAnchorsMap::const_iterator it = cacheAnchors.find(rt); + if (it != cacheAnchors.end()) { + if (it->second.entered) { + tree.setTo(it->second.tree); + return true; + } else { + return false; + } + } + + CAnchorsCacheEntry entry; + if (!base->GetAnchorAt(rt, tree)) { + return false; + } + + entry.entered = true; + entry.tree.setTo(tree); + cacheAnchors.insert(std::make_pair(rt, entry)); + + return true; +} + + +void CCoinsViewCache::PushAnchor(const libzerocash::IncrementalMerkleTree &tree) { + std::vector newrt_v(32); + tree.getRootValue(newrt_v); + uint256 newrt(newrt_v); + + auto currentRoot = GetBestAnchor(); + + // We don't want to overwrite an anchor we already have. + // This occurs when a block doesn't modify mapAnchors at all, + // because there are no pours. We could get around this a + // different way (make all blocks modify mapAnchors somehow) + // but this is simpler to reason about. + if (currentRoot != newrt) { + CAnchorsMap::iterator ret = cacheAnchors.insert(std::make_pair(newrt, CAnchorsCacheEntry())).first; + + ret->second.entered = true; + ret->second.tree.setTo(tree); + ret->second.flags = CAnchorsCacheEntry::DIRTY; + + hashAnchor = newrt; + } +} + +void CCoinsViewCache::PopAnchor(const uint256 &newrt) { + auto currentRoot = GetBestAnchor(); + + // Blocks might not change the commitment tree, in which + // case restoring the "old" anchor during a reorg must + // have no effect. + if (currentRoot != newrt) { + CAnchorsMap::iterator ret = cacheAnchors.insert(std::make_pair(currentRoot, CAnchorsCacheEntry())).first; + + ret->second.entered = false; + ret->second.flags = CAnchorsCacheEntry::DIRTY; + + hashAnchor = newrt; + } +} + bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { CCoinsMap::const_iterator it = FetchCoins(txid); if (it != cacheCoins.end()) { @@ -141,11 +215,21 @@ uint256 CCoinsViewCache::GetBestBlock() const { return hashBlock; } + +uint256 CCoinsViewCache::GetBestAnchor() const { + if (hashAnchor.IsNull()) + hashAnchor = base->GetBestAnchor(); + return hashAnchor; +} + void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { hashBlock = hashBlockIn; } -bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { +bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlockIn, + const uint256 &hashAnchorIn, + CAnchorsMap &mapAnchors) { assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). @@ -181,13 +265,44 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } + + for (CAnchorsMap::iterator child_it = mapAnchors.begin(); child_it != mapAnchors.end();) + { + if (child_it->second.flags & CAnchorsCacheEntry::DIRTY) { + CAnchorsMap::iterator parent_it = cacheAnchors.find(child_it->first); + + if (parent_it == cacheAnchors.end()) { + if (child_it->second.entered) { + // Parent doesn't have an entry, but child has a new commitment root. + + CAnchorsCacheEntry& entry = cacheAnchors[child_it->first]; + entry.entered = true; + entry.tree.setTo(child_it->second.tree); + entry.flags = CAnchorsCacheEntry::DIRTY; + + // TODO: cache usage + } + } else { + if (parent_it->second.entered != child_it->second.entered) { + // The parent may have removed the entry. + parent_it->second.entered = child_it->second.entered; + parent_it->second.flags |= CAnchorsCacheEntry::DIRTY; + } + } + } + + CAnchorsMap::iterator itOld = child_it++; + mapAnchors.erase(itOld); + } + hashAnchor = hashAnchorIn; hashBlock = hashBlockIn; return true; } bool CCoinsViewCache::Flush() { - bool fOk = base->BatchWrite(cacheCoins, hashBlock); + bool fOk = base->BatchWrite(cacheCoins, hashBlock, hashAnchor, cacheAnchors); cacheCoins.clear(); + cacheAnchors.clear(); cachedCoinsUsage = 0; return fOk; } diff --git a/src/coins.h b/src/coins.h index a4671645d..2c761aca6 100644 --- a/src/coins.h +++ b/src/coins.h @@ -16,6 +16,9 @@ #include #include +#include "libzerocash/IncrementalMerkleTree.h" + +static const unsigned int INCREMENTAL_MERKLE_TREE_DEPTH = 20; /** * Pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -295,7 +298,21 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; +struct CAnchorsCacheEntry +{ + bool entered; // This will be false if the anchor is removed from the cache + libzerocash::IncrementalMerkleTree tree; // The tree itself + unsigned char flags; + + enum Flags { + DIRTY = (1 << 0), // This cache entry is potentially different from the version in the parent view. + }; + + CAnchorsCacheEntry() : entered(false), flags(0), tree(INCREMENTAL_MERKLE_TREE_DEPTH) {} +}; + typedef boost::unordered_map CCoinsMap; +typedef boost::unordered_map CAnchorsMap; struct CCoinsStats { @@ -315,6 +332,9 @@ struct CCoinsStats class CCoinsView { public: + //! Retrieve the tree at a particular anchored root in the chain + virtual bool GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const; + //! Retrieve the CCoins (unspent transaction outputs) for a given txid virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; @@ -325,9 +345,15 @@ public: //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; + //! Get the current "tip" or the latest anchored tree root in the chain + virtual uint256 GetBestAnchor() const; + //! Do a bulk modification (multiple CCoins changes + BestBlock change). //! The passed mapCoins can be modified. - virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + virtual bool BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors); //! Calculate statistics about the unspent transaction output set virtual bool GetStats(CCoinsStats &stats) const; @@ -345,11 +371,16 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); + bool GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const; bool GetCoins(const uint256 &txid, CCoins &coins) const; bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; + uint256 GetBestAnchor() const; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + bool BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors); bool GetStats(CCoinsStats &stats) const; }; @@ -390,6 +421,8 @@ protected: */ mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; + mutable uint256 hashAnchor; + mutable CAnchorsMap cacheAnchors; /* Cached dynamic memory usage for the inner CCoins objects. */ mutable size_t cachedCoinsUsage; @@ -399,11 +432,25 @@ public: ~CCoinsViewCache(); // Standard CCoinsView methods + bool GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const; bool GetCoins(const uint256 &txid, CCoins &coins) const; bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; + uint256 GetBestAnchor() const; void SetBestBlock(const uint256 &hashBlock); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + bool BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors); + + + // Adds the tree to mapAnchors and sets the current commitment + // root to this root. + void PushAnchor(const libzerocash::IncrementalMerkleTree &tree); + + // Removes the current commitment root from mapAnchors and sets + // the new current root. + void PopAnchor(const uint256 &rt); /** * Return a pointer to CCoins in the cache, or NULL if not found. This is diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 34b311b80..4ec89b7df 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -11,15 +11,36 @@ #include #include +#include "libzerocash/IncrementalMerkleTree.h" namespace { class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; + uint256 hashBestAnchor_; std::map map_; + std::map mapAnchors_; public: + bool GetAnchorAt(const uint256& rt, libzerocash::IncrementalMerkleTree &tree) const { + if (rt.IsNull()) { + IncrementalMerkleTree new_tree(INCREMENTAL_MERKLE_TREE_DEPTH); + tree.setTo(new_tree); + return true; + } + + std::map::const_iterator it = mapAnchors_.find(rt); + if (it == mapAnchors_.end()) { + return false; + } else { + tree.setTo(it->second); + return true; + } + } + + uint256 GetBestAnchor() const { return hashBestAnchor_; } + bool GetCoins(const uint256& txid, CCoins& coins) const { std::map::const_iterator it = map_.find(txid); @@ -42,7 +63,10 @@ public: uint256 GetBestBlock() const { return hashBestBlock_; } - bool BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock) + bool BatchWrite(CCoinsMap& mapCoins, + const uint256& hashBlock, + const uint256& hashAnchor, + CAnchorsMap& mapAnchors) { for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { map_[it->first] = it->second.coins; @@ -52,8 +76,18 @@ public: } mapCoins.erase(it++); } + for (CAnchorsMap::iterator it = mapAnchors.begin(); it != mapAnchors.end(); ) { + if (it->second.entered) { + mapAnchors_[it->first] = it->second.tree; + } else { + mapAnchors_.erase(it->first); + } + mapAnchors.erase(it++); + } mapCoins.clear(); + mapAnchors.clear(); hashBestBlock_ = hashBlock; + hashBestAnchor_ = hashAnchor; return true; } @@ -79,6 +113,133 @@ public: } +void appendRandomCommitment(IncrementalMerkleTree &tree) +{ + Address addr = Address::CreateNewRandomAddress(); + Coin coin(addr.getPublicAddress(), 100); + + std::vector commitment(ZC_CM_SIZE * 8); + convertBytesVectorToVector(coin.getCoinCommitment().getCommitmentValue(), commitment); + + std::vector index; + tree.insertElement(commitment, index); +} + +BOOST_AUTO_TEST_CASE(anchors_test) +{ + // TODO: These tests should be more methodical. + // Or, integrate with Bitcoin's tests later. + + CCoinsViewTest base; + CCoinsViewCacheTest cache(&base); + + BOOST_CHECK(cache.GetBestAnchor() == uint256()); + + { + IncrementalMerkleTree tree(INCREMENTAL_MERKLE_TREE_DEPTH); + + BOOST_CHECK(cache.GetAnchorAt(cache.GetBestAnchor(), tree)); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + appendRandomCommitment(tree); + tree.prune(); + + IncrementalMerkleTree save_tree_for_later(INCREMENTAL_MERKLE_TREE_DEPTH); + save_tree_for_later.setTo(tree); + + uint256 newrt; + uint256 newrt2; + { + std::vector newrt_v(32); + tree.getRootValue(newrt_v); + + newrt = uint256(newrt_v); + } + + cache.PushAnchor(tree); + BOOST_CHECK(cache.GetBestAnchor() == newrt); + + { + IncrementalMerkleTree confirm_same(INCREMENTAL_MERKLE_TREE_DEPTH); + BOOST_CHECK(cache.GetAnchorAt(cache.GetBestAnchor(), confirm_same)); + + uint256 confirm_rt; + { + std::vector newrt_v(32); + confirm_same.getRootValue(newrt_v); + + confirm_rt = uint256(newrt_v); + } + + BOOST_CHECK(confirm_rt == newrt); + } + + appendRandomCommitment(tree); + appendRandomCommitment(tree); + tree.prune(); + + { + std::vector newrt_v(32); + tree.getRootValue(newrt_v); + + newrt2 = uint256(newrt_v); + } + + cache.PushAnchor(tree); + BOOST_CHECK(cache.GetBestAnchor() == newrt2); + + IncrementalMerkleTree test_tree(INCREMENTAL_MERKLE_TREE_DEPTH); + BOOST_CHECK(cache.GetAnchorAt(cache.GetBestAnchor(), test_tree)); + + { + std::vector a(32); + std::vector b(32); + tree.getRootValue(a); + test_tree.getRootValue(b); + + BOOST_CHECK(a == b); + } + + { + std::vector a(32); + std::vector b(32); + IncrementalMerkleTree test_tree2(INCREMENTAL_MERKLE_TREE_DEPTH); + cache.GetAnchorAt(newrt, test_tree2); + + uint256 recovered_rt; + { + std::vector newrt_v(32); + test_tree2.getRootValue(newrt_v); + + recovered_rt = uint256(newrt_v); + } + + BOOST_CHECK(recovered_rt == newrt); + } + + { + cache.PopAnchor(newrt); + IncrementalMerkleTree obtain_tree(INCREMENTAL_MERKLE_TREE_DEPTH); + assert(!cache.GetAnchorAt(newrt2, obtain_tree)); // should have been popped off + assert(cache.GetAnchorAt(newrt, obtain_tree)); + + uint256 recovered_rt; + { + std::vector newrt_v(32); + obtain_tree.getRootValue(newrt_v); + + recovered_rt = uint256(newrt_v); + } + + assert(recovered_rt == newrt); + } + } +} + BOOST_FIXTURE_TEST_SUITE(coins_tests, BasicTestingSetup) static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; diff --git a/src/txdb.cpp b/src/txdb.cpp index df9ff8d8c..aa4070aa1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -17,17 +17,31 @@ using namespace std; +static const char DB_ANCHOR = 'A'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; +static const char DB_BEST_ANCHOR = 'a'; static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +void static BatchWriteAnchor(CLevelDBBatch &batch, + const uint256 &croot, + const libzerocash::IncrementalMerkleTree &tree, + const bool &entered) +{ + if (!entered) + batch.Erase(make_pair(DB_ANCHOR, croot)); + else { + batch.Write(make_pair(DB_ANCHOR, croot), tree.serialize()); + } +} + void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) { if (coins.IsPruned()) batch.Erase(make_pair(DB_COINS, hash)); @@ -39,9 +53,34 @@ void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) { batch.Write(DB_BEST_BLOCK, hash); } +void static BatchWriteHashBestAnchor(CLevelDBBatch &batch, const uint256 &hash) { + batch.Write(DB_BEST_ANCHOR, hash); +} + CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) { } + +bool CCoinsViewDB::GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const { + if (rt.IsNull()) { + IncrementalMerkleTree new_tree(INCREMENTAL_MERKLE_TREE_DEPTH); + tree.setTo(new_tree); + return true; + } + + std::vector tree_serialized; + + bool read = db.Read(make_pair(DB_ANCHOR, rt), tree_serialized); + + if (!read) return read; + + auto tree_deserialized = IncrementalMerkleTreeCompact::deserialize(tree_serialized); + + tree.fromCompactRepresentation(tree_deserialized); + + return true; +} + bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { return db.Read(make_pair(DB_COINS, txid), coins); } @@ -57,7 +96,17 @@ uint256 CCoinsViewDB::GetBestBlock() const { return hashBestChain; } -bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { +uint256 CCoinsViewDB::GetBestAnchor() const { + uint256 hashBestAnchor; + if (!db.Read(DB_BEST_ANCHOR, hashBestAnchor)) + return uint256(); + return hashBestAnchor; +} + +bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors) { CLevelDBBatch batch; size_t count = 0; size_t changed = 0; @@ -70,8 +119,20 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); } + + for (CAnchorsMap::iterator it = mapAnchors.begin(); it != mapAnchors.end();) { + if (it->second.flags & CAnchorsCacheEntry::DIRTY) { + BatchWriteAnchor(batch, it->first, it->second.tree, it->second.entered); + // TODO: changed++? + } + CAnchorsMap::iterator itOld = it++; + mapAnchors.erase(itOld); + } + if (!hashBlock.IsNull()) BatchWriteHashBestChain(batch, hashBlock); + if (!hashAnchor.IsNull()) + BatchWriteHashBestAnchor(batch, hashAnchor); LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); diff --git a/src/txdb.h b/src/txdb.h index bef5dc9fd..9980d6eb4 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -14,6 +14,8 @@ #include #include +#include "libzerocash/IncrementalMerkleTree.h" + class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; @@ -34,10 +36,15 @@ protected: public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + bool GetAnchorAt(const uint256 &rt, libzerocash::IncrementalMerkleTree &tree) const; bool GetCoins(const uint256 &txid, CCoins &coins) const; bool HaveCoins(const uint256 &txid) const; uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + uint256 GetBestAnchor() const; + bool BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors); bool GetStats(CCoinsStats &stats) const; };