From 0d81464be78ce83545132dfdf1c359868063cc8f Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Mon, 7 Sep 2015 15:22:23 -0700 Subject: [PATCH 01/15] Refactor leveldbwrapper Was "Add chainstate obfuscation to avoid spurious antivirus detection" Zcash: Extracted the refactor, omitting the chainstate obfuscation. --- src/Makefile.test.include | 1 + src/leveldbwrapper.cpp | 8 ++++++ src/leveldbwrapper.h | 13 +++++++++ src/test/leveldbwrapper_tests.cpp | 45 +++++++++++++++++++++++++++++++ src/txdb.cpp | 21 +++++---------- 5 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 src/test/leveldbwrapper_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 582ecb577..02ae5ad3e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -60,6 +60,7 @@ BITCOIN_TESTS =\ test/getarg_tests.cpp \ test/hash_tests.cpp \ test/key_tests.cpp \ + test/leveldbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index b5d024abb..22658b092 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -12,6 +12,7 @@ #include #include #include +#include void HandleError(const leveldb::Status& status) { @@ -87,3 +88,10 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) HandleError(status); return true; } + +bool CLevelDBWrapper::IsEmpty() +{ + boost::scoped_ptr it(NewIterator()); + it->SeekToFirst(); + return !(it->Valid()); +} diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index 639f736a5..2bcef7daf 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -86,6 +86,12 @@ private: leveldb::DB* pdb; public: + /** + * @param[in] path Location in the filesystem where leveldb data will be stored. + * @param[in] nCacheSize Configures various leveldb cache settings. + * @param[in] fMemory If true, use leveldb's memory environment. + * @param[in] fWipe If true, remove all existing data. + */ CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); ~CLevelDBWrapper(); @@ -168,6 +174,13 @@ public: { return pdb->NewIterator(iteroptions); } + + /** + * Return true if the database managed by this class contains no entries. + */ + bool IsEmpty(); + }; #endif // BITCOIN_LEVELDBWRAPPER_H + diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp new file mode 100644 index 000000000..3fa74573f --- /dev/null +++ b/src/test/leveldbwrapper_tests.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2012-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "leveldbwrapper.h" +#include "uint256.h" +#include "random.h" +#include "test/test_bitcoin.h" + +#include // for 'operator+=()' +#include +#include + +using namespace std; +using namespace boost::assign; // bring 'operator+=()' into scope +using namespace boost::filesystem; + +// Test if a string consists entirely of null characters +bool is_null_key(const vector& key) { + bool isnull = true; + + for (unsigned int i = 0; i < key.size(); i++) + isnull &= (key[i] == '\x00'); + + return isnull; +} + +BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(leveldbwrapper) +{ + { + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false); + char key = 'k'; + uint256 in = GetRandHash(); + uint256 res; + + BOOST_CHECK(dbw.Write(key, in)); + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index a40df59aa..3ca19a356 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -50,17 +50,6 @@ void static BatchWriteNullifier(CLevelDBBatch &batch, const uint256 &nf, const b batch.Write(make_pair(DB_NULLIFIER, nf), true); } -void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) { - if (coins.IsPruned()) - batch.Erase(make_pair(DB_COINS, hash)); - else - batch.Write(make_pair(DB_COINS, hash), coins); -} - -void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) { - batch.Write(DB_BEST_BLOCK, hash); -} - void static BatchWriteHashBestAnchor(CLevelDBBatch &batch, const uint256 &hash) { batch.Write(DB_BEST_ANCHOR, hash); } @@ -68,7 +57,8 @@ void static BatchWriteHashBestAnchor(CLevelDBBatch &batch, const uint256 &hash) CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe) { } -CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) { +CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) +{ } @@ -123,7 +113,10 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - BatchWriteCoins(batch, it->first, it->second.coins); + if (it->second.coins.IsPruned()) + batch.Erase(make_pair(DB_COINS, it->first)); + else + batch.Write(make_pair(DB_COINS, it->first), it->second.coins); changed++; } count++; @@ -150,7 +143,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, } if (!hashBlock.IsNull()) - BatchWriteHashBestChain(batch, hashBlock); + batch.Write(DB_BEST_BLOCK, hashBlock); if (!hashAnchor.IsNull()) BatchWriteHashBestAnchor(batch, hashAnchor); From 3a8e1d0cf43ae3d3eae2943dc8d0fd085cb31e3d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2017 14:02:27 +0100 Subject: [PATCH 02/15] Refactor Zcash changes to CCoinsViewDB To match upstream changes. --- src/txdb.cpp | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 3ca19a356..297777781 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -31,29 +31,6 @@ static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; -void static BatchWriteAnchor(CLevelDBBatch &batch, - const uint256 &croot, - const ZCIncrementalMerkleTree &tree, - const bool &entered) -{ - if (!entered) - batch.Erase(make_pair(DB_ANCHOR, croot)); - else { - batch.Write(make_pair(DB_ANCHOR, croot), tree); - } -} - -void static BatchWriteNullifier(CLevelDBBatch &batch, const uint256 &nf, const bool &entered) { - if (!entered) - batch.Erase(make_pair(DB_NULLIFIER, nf)); - else - batch.Write(make_pair(DB_NULLIFIER, nf), true); -} - -void static BatchWriteHashBestAnchor(CLevelDBBatch &batch, const uint256 &hash) { - batch.Write(DB_BEST_ANCHOR, hash); -} - CCoinsViewDB::CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / dbName, nCacheSize, fMemory, fWipe) { } @@ -126,7 +103,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, for (CAnchorsMap::iterator it = mapAnchors.begin(); it != mapAnchors.end();) { if (it->second.flags & CAnchorsCacheEntry::DIRTY) { - BatchWriteAnchor(batch, it->first, it->second.tree, it->second.entered); + if (!it->second.entered) + batch.Erase(make_pair(DB_ANCHOR, it->first)); + else { + batch.Write(make_pair(DB_ANCHOR, it->first), it->second.tree); + } // TODO: changed++? } CAnchorsMap::iterator itOld = it++; @@ -135,7 +116,10 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, for (CNullifiersMap::iterator it = mapNullifiers.begin(); it != mapNullifiers.end();) { if (it->second.flags & CNullifiersCacheEntry::DIRTY) { - BatchWriteNullifier(batch, it->first, it->second.entered); + if (!it->second.entered) + batch.Erase(make_pair(DB_NULLIFIER, it->first)); + else + batch.Write(make_pair(DB_NULLIFIER, it->first), true); // TODO: changed++? } CNullifiersMap::iterator itOld = it++; @@ -145,7 +129,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); if (!hashAnchor.IsNull()) - BatchWriteHashBestAnchor(batch, hashAnchor); + batch.Write(DB_BEST_ANCHOR, hashAnchor); LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); From 1ebf50b6dada3bace0ed3e02266f326096c98831 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 7 Oct 2015 17:12:24 -0700 Subject: [PATCH 03/15] Encapsulate CLevelDB iterators cleanly Conflicts: src/leveldb.cpp src/leveldb.h src/txdb.cpp --- src/leveldbwrapper.cpp | 9 +++++- src/leveldbwrapper.h | 64 +++++++++++++++++++++++++++++++++++++++--- src/txdb.cpp | 59 +++++++++++++------------------------- 3 files changed, 88 insertions(+), 44 deletions(-) diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index 22658b092..2a3e777c9 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -91,7 +91,14 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) bool CLevelDBWrapper::IsEmpty() { - boost::scoped_ptr it(NewIterator()); + boost::scoped_ptr it(NewIterator()); it->SeekToFirst(); return !(it->Valid()); } + +CLevelDBIterator::~CLevelDBIterator() { delete piter; } +bool CLevelDBIterator::Valid() { return piter->Valid(); } +void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); } +void CLevelDBIterator::Next() { piter->Next(); } +void CLevelDBIterator::Prev() { piter->Prev(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index 2bcef7daf..daa0f78ce 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -60,7 +60,64 @@ public: batch.Delete(slKey); } }; + +class CLevelDBIterator +{ +private: + leveldb::Iterator *piter; +public: + CLevelDBIterator(leveldb::Iterator *piterIn) : piter(piterIn) {} + ~CLevelDBIterator(); + + bool Valid(); + + void SeekToFirst(); + void SeekToLast(); + + template void Seek(const K& key) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(ssKey.GetSerializeSize(key)); + ssKey << key; + leveldb::Slice slKey(&ssKey[0], ssKey.size()); + piter->Seek(slKey); + } + + void Next(); + void Prev(); + + template bool GetKey(K& key) { + leveldb::Slice slKey = piter->key(); + try { + CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + ssKey >> key; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetKeySize() { + return piter->key().size(); + } + + template bool GetValue(V& value) { + leveldb::Slice slValue = piter->value(); + try { + CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + ssValue >> value; + } catch(std::exception &e) { + return false; + } + return true; + } + + unsigned int GetValueSize() { + return piter->value().size(); + } + +}; + class CLevelDBWrapper { private: @@ -169,11 +226,10 @@ public: return WriteBatch(batch, true); } - // not exactly clean encapsulation, but it's easiest for now - leveldb::Iterator* NewIterator() + CLevelDBIterator *NewIterator() + { + return new CLevelDBIterator(pdb->NewIterator(iteroptions)); { - return pdb->NewIterator(iteroptions); - } /** * Return true if the database managed by this class contains no entries. diff --git a/src/txdb.cpp b/src/txdb.cpp index 297777781..29b49745c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -162,8 +162,8 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->SeekToFirst(); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -171,22 +171,10 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { CAmount nTotalAmount = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_COINS) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CCoins coins; - ssValue >> coins; - uint256 txhash; - ssKey >> txhash; - ss << txhash; - ss << VARINT(coins.nVersion); - ss << (coins.fCoinBase ? 'c' : 'n'); - ss << VARINT(coins.nHeight); + std::pair key; + CCoins coins; + if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; iGetKeySize(); ss << VARINT(0); + } else { + return error("CCoinsViewDB::GetStats() : unable to read value"); } - pcursor->Next(); - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } + pcursor->Next(); } { LOCK(cs_main); @@ -261,24 +251,15 @@ bool CBlockTreeDB::LoadBlockIndexGuts() { boost::scoped_ptr pcursor(NewIterator()); - CDataStream ssKeySet(SER_DISK, CLIENT_VERSION); - ssKeySet << make_pair(DB_BLOCK_INDEX, uint256()); - pcursor->Seek(ssKeySet.str()); + pcursor->Seek(make_pair('b', uint256(0))); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); - try { - leveldb::Slice slKey = pcursor->key(); - CDataStream ssKey(slKey.data(), slKey.data()+slKey.size(), SER_DISK, CLIENT_VERSION); - char chType; - ssKey >> chType; - if (chType == DB_BLOCK_INDEX) { - leveldb::Slice slValue = pcursor->value(); - CDataStream ssValue(slValue.data(), slValue.data()+slValue.size(), SER_DISK, CLIENT_VERSION); - CDiskBlockIndex diskindex; - ssValue >> diskindex; - + std::pair key; + if (pcursor->GetKey(key) && key.first == 'b') { + CDiskBlockIndex diskindex; + if (pcursor->GetValue(diskindex)) { // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); @@ -309,10 +290,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts() pcursor->Next(); } else { - break; // if shutdown requested or finished loading block index + return error("LoadBlockIndex() : failed to read value"); } - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } else { + break; } } From 0d9524ba6b32c11f09fa77279091ed1d6433ae18 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 8 Oct 2015 00:44:10 -0700 Subject: [PATCH 04/15] Minor bugfixes Was "Handle obfuscation in CLevelDBIterator" Zcash: Only bugfixes --- src/leveldbwrapper.cpp | 2 +- src/leveldbwrapper.h | 9 +++++++-- src/txdb.cpp | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index 2a3e777c9..ace633c6a 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -95,7 +95,7 @@ bool CLevelDBWrapper::IsEmpty() it->SeekToFirst(); return !(it->Valid()); } - + CLevelDBIterator::~CLevelDBIterator() { delete piter; } bool CLevelDBIterator::Valid() { return piter->Valid(); } void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index daa0f78ce..e422a6e01 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -67,7 +67,12 @@ private: leveldb::Iterator *piter; public: - CLevelDBIterator(leveldb::Iterator *piterIn) : piter(piterIn) {} + + /** + * @param[in] piterIn The original leveldb iterator. + */ + CLevelDBIterator(leveldb::Iterator *piterIn) : + piter(piterIn) { }; ~CLevelDBIterator(); bool Valid(); @@ -229,7 +234,7 @@ public: CLevelDBIterator *NewIterator() { return new CLevelDBIterator(pdb->NewIterator(iteroptions)); - { + } /** * Return true if the database managed by this class contains no entries. diff --git a/src/txdb.cpp b/src/txdb.cpp index 29b49745c..b416b1ad3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -162,7 +162,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->Seek('c'); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -249,9 +249,9 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair('b', uint256(0))); + pcursor->Seek(make_pair('b', uint256())); // Load mapBlockIndex while (pcursor->Valid()) { From 79272bfaf9686c208c3b2d6c6eafef91526adabd Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 8 Oct 2015 01:22:50 -0700 Subject: [PATCH 05/15] Add tests for gettxoutsetinfo, CLevelDBBatch, CLevelDBIterator Thanks @dexX7. --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/blockchain.py | 52 +++++++++++++++++++++ src/test/leveldbwrapper_tests.cpp | 75 +++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100755 qa/rpc-tests/blockchain.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index e1b7a71b4..07789a03a 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -45,6 +45,7 @@ testScripts=( 'nodehandling.py' 'reindex.py' 'decodescript.py' + 'blockchain.py' 'disablewallet.py' 'zcjoinsplit.py' 'zcjoinsplitdoublespend.py' diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py new file mode 100755 index 000000000..a5c98b777 --- /dev/null +++ b/qa/rpc-tests/blockchain.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test RPC calls related to blockchain state. +# + +import decimal + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + initialize_chain, + assert_equal, + start_nodes, + connect_nodes_bi, +) + +class BlockchainTest(BitcoinTestFramework): + """ + Test blockchain-related RPC calls: + + - gettxoutsetinfo + + """ + + def setup_chain(self): + print("Initializing test directory " + self.options.tmpdir) + initialize_chain(self.options.tmpdir) + + def setup_network(self, split=False): + self.nodes = start_nodes(2, self.options.tmpdir) + connect_nodes_bi(self.nodes, 0, 1) + self.is_network_split = False + self.sync_all() + + def run_test(self): + node = self.nodes[0] + res = node.gettxoutsetinfo() + + assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000')) + assert_equal(res[u'transactions'], 200) + assert_equal(res[u'height'], 200) + assert_equal(res[u'txouts'], 200) + assert_equal(res[u'bytes_serialized'], 13000), + assert_equal(len(res[u'bestblock']), 64) + assert_equal(len(res[u'hash_serialized']), 64) + + +if __name__ == '__main__': + BlockchainTest().main() diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp index 3fa74573f..6ea1a515a 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/leveldbwrapper_tests.cpp @@ -41,5 +41,80 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); } } + +// Test batch operations +BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) +{ + { + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false); + + char key = 'i'; + uint256 in = GetRandHash(); + char key2 = 'j'; + uint256 in2 = GetRandHash(); + char key3 = 'k'; + uint256 in3 = GetRandHash(); + + uint256 res; + CLevelDBBatch batch; + + batch.Write(key, in); + batch.Write(key2, in2); + batch.Write(key3, in3); + + // Remove key3 before it's even been written + batch.Erase(key3); + + dbw.WriteBatch(batch); + + BOOST_CHECK(dbw.Read(key, res)); + BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); + BOOST_CHECK(dbw.Read(key2, res)); + BOOST_CHECK_EQUAL(res.ToString(), in2.ToString()); + + // key3 never should've been written + BOOST_CHECK(dbw.Read(key3, res) == false); + } +} + +BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) +{ + { + path ph = temp_directory_path() / unique_path(); + CLevelDBWrapper dbw(ph, (1 << 20), true, false); + + // The two keys are intentionally chosen for ordering + char key = 'j'; + uint256 in = GetRandHash(); + BOOST_CHECK(dbw.Write(key, in)); + char key2 = 'k'; + uint256 in2 = GetRandHash(); + BOOST_CHECK(dbw.Write(key2, in2)); + + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + + // Be sure to seek past any earlier key (if it exists) + it->Seek(key); + + char key_res; + uint256 val_res; + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key); + BOOST_CHECK_EQUAL(val_res.ToString(), in.ToString()); + + it->Next(); + + it->GetKey(key_res); + it->GetValue(val_res); + BOOST_CHECK_EQUAL(key_res, key2); + BOOST_CHECK_EQUAL(val_res.ToString(), in2.ToString()); + + it->Next(); + BOOST_CHECK_EQUAL(it->Valid(), false); + } +} BOOST_AUTO_TEST_SUITE_END() From f95bf4c417d6bf222b628029cc622662b1402720 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 21 Oct 2015 23:23:59 +0200 Subject: [PATCH 06/15] Fix chainstate serialized_size computation --- qa/rpc-tests/blockchain.py | 2 +- src/txdb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index a5c98b777..b7bfe3628 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -43,7 +43,7 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res[u'transactions'], 200) assert_equal(res[u'height'], 200) assert_equal(res[u'txouts'], 200) - assert_equal(res[u'bytes_serialized'], 13000), + assert_equal(res[u'bytes_serialized'], 13924), assert_equal(len(res[u'bestblock']), 64) assert_equal(len(res[u'hash_serialized']), 64) diff --git a/src/txdb.cpp b/src/txdb.cpp index b416b1ad3..25e954c8d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -185,7 +185,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { nTotalAmount += out.nValue; } } - stats.nSerializedSize += 32 + pcursor->GetKeySize(); + stats.nSerializedSize += 32 + pcursor->GetValueSize(); ss << VARINT(0); } else { return error("CCoinsViewDB::GetStats() : unable to read value"); From 26cdb83add812c4e53be56688f243275ed58e84c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Mon, 28 Aug 2017 16:43:46 +0100 Subject: [PATCH 07/15] Update blockchain.py RPC test for Zcash --- qa/rpc-tests/blockchain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index b7bfe3628..85b061317 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -39,11 +39,11 @@ class BlockchainTest(BitcoinTestFramework): node = self.nodes[0] res = node.gettxoutsetinfo() - assert_equal(res[u'total_amount'], decimal.Decimal('8725.00000000')) + assert_equal(res[u'total_amount'], decimal.Decimal('2181.25000000')) # 150*12.5 + 49*6.25 assert_equal(res[u'transactions'], 200) assert_equal(res[u'height'], 200) - assert_equal(res[u'txouts'], 200) - assert_equal(res[u'bytes_serialized'], 13924), + assert_equal(res[u'txouts'], 349) # 150*2 + 49 + assert_equal(res[u'bytes_serialized'], 14951), # 32*199 + 48*90 + 49*60 + 27*49 assert_equal(len(res[u'bestblock']), 64) assert_equal(len(res[u'hash_serialized']), 64) From c77586964a123608d0ec4e5201f023c34b4fdc57 Mon Sep 17 00:00:00 2001 From: Daniel Kraft Date: Tue, 13 Oct 2015 20:25:57 +0200 Subject: [PATCH 08/15] trivial: use constants for db keys Replace literal occurances of the key "prefixes" 'c' and 'b' in txdb.cpp by the respective constants. --- src/txdb.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 25e954c8d..59a3ba4ca 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -163,7 +163,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { only need read operations on it, use a const-cast to get around that restriction. */ boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); - pcursor->Seek('c'); + pcursor->Seek(DB_COINS); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = GetBestBlock(); @@ -173,7 +173,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { boost::this_thread::interruption_point(); std::pair key; CCoins coins; - if (pcursor->GetKey(key) && key.first == 'c') { + if (pcursor->GetKey(key) && key.first == DB_COINS) { if (pcursor->GetValue(coins)) { stats.nTransactions++; for (unsigned int i=0; i pcursor(NewIterator()); - pcursor->Seek(make_pair('b', uint256())); + pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); // Load mapBlockIndex while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; - if (pcursor->GetKey(key) && key.first == 'b') { + if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) { CDiskBlockIndex diskindex; if (pcursor->GetValue(diskindex)) { // Construct block index object From 7249ee6d7cb9b4a56120a403fc02f96a336d3b20 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 22 Oct 2015 20:49:02 -0400 Subject: [PATCH 09/15] leveldbwrapper: Remove unused .Prev(), .SeekToLast() methods Also, trim trailing whitespace. --- src/leveldbwrapper.cpp | 2 -- src/leveldbwrapper.h | 8 +++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index ace633c6a..492623e45 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -99,6 +99,4 @@ bool CLevelDBWrapper::IsEmpty() CLevelDBIterator::~CLevelDBIterator() { delete piter; } bool CLevelDBIterator::Valid() { return piter->Valid(); } void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CLevelDBIterator::SeekToLast() { piter->SeekToLast(); } void CLevelDBIterator::Next() { piter->Next(); } -void CLevelDBIterator::Prev() { piter->Prev(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index e422a6e01..d3d8e6322 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -60,7 +60,7 @@ public: batch.Delete(slKey); } }; - + class CLevelDBIterator { private: @@ -78,7 +78,6 @@ public: bool Valid(); void SeekToFirst(); - void SeekToLast(); template void Seek(const K& key) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); @@ -89,7 +88,6 @@ public: } void Next(); - void Prev(); template bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); @@ -122,7 +120,7 @@ public: } }; - + class CLevelDBWrapper { private: @@ -231,7 +229,7 @@ public: return WriteBatch(batch, true); } - CLevelDBIterator *NewIterator() + CLevelDBIterator *NewIterator() { return new CLevelDBIterator(pdb->NewIterator(iteroptions)); } From f345c41ec4900cbba09afe403e12e3039c9f0f2f Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 22 Oct 2015 21:02:20 -0400 Subject: [PATCH 10/15] leveldbwrapper symbol rename: Remove "Level" from class, etc. names --- src/leveldbwrapper.cpp | 26 +++++++++++------------ src/leveldbwrapper.h | 34 +++++++++++++++---------------- src/test/leveldbwrapper_tests.cpp | 18 ++++++++-------- src/txdb.cpp | 12 +++++------ src/txdb.h | 6 +++--- 5 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index 492623e45..9933df7e4 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -20,12 +20,12 @@ void HandleError(const leveldb::Status& status) return; LogPrintf("%s\n", status.ToString()); if (status.IsCorruption()) - throw leveldb_error("Database corrupted"); + throw dbwrapper_error("Database corrupted"); if (status.IsIOError()) - throw leveldb_error("Database I/O error"); + throw dbwrapper_error("Database I/O error"); if (status.IsNotFound()) - throw leveldb_error("Database entry missing"); - throw leveldb_error("Unknown database error"); + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); } static leveldb::Options GetOptions(size_t nCacheSize) @@ -44,7 +44,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) return options; } -CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe) +CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe) { penv = NULL; readoptions.verify_checksums = true; @@ -70,7 +70,7 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa LogPrintf("Opened LevelDB successfully\n"); } -CLevelDBWrapper::~CLevelDBWrapper() +CDBWrapper::~CDBWrapper() { delete pdb; pdb = NULL; @@ -82,21 +82,21 @@ CLevelDBWrapper::~CLevelDBWrapper() options.env = NULL; } -bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) +bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); HandleError(status); return true; } -bool CLevelDBWrapper::IsEmpty() +bool CDBWrapper::IsEmpty() { - boost::scoped_ptr it(NewIterator()); + boost::scoped_ptr it(NewIterator()); it->SeekToFirst(); return !(it->Valid()); } -CLevelDBIterator::~CLevelDBIterator() { delete piter; } -bool CLevelDBIterator::Valid() { return piter->Valid(); } -void CLevelDBIterator::SeekToFirst() { piter->SeekToFirst(); } -void CLevelDBIterator::Next() { piter->Next(); } +CDBIterator::~CDBIterator() { delete piter; } +bool CDBIterator::Valid() { return piter->Valid(); } +void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } +void CDBIterator::Next() { piter->Next(); } diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index d3d8e6322..b7f6fa38c 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -16,18 +16,18 @@ #include #include -class leveldb_error : public std::runtime_error +class dbwrapper_error : public std::runtime_error { public: - leveldb_error(const std::string& msg) : std::runtime_error(msg) {} + dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} }; void HandleError(const leveldb::Status& status); -/** Batch of changes queued to be written to a CLevelDBWrapper */ -class CLevelDBBatch +/** Batch of changes queued to be written to a CDBWrapper */ +class CDBBatch { - friend class CLevelDBWrapper; + friend class CDBWrapper; private: leveldb::WriteBatch batch; @@ -61,7 +61,7 @@ public: } }; -class CLevelDBIterator +class CDBIterator { private: leveldb::Iterator *piter; @@ -71,9 +71,9 @@ public: /** * @param[in] piterIn The original leveldb iterator. */ - CLevelDBIterator(leveldb::Iterator *piterIn) : + CDBIterator(leveldb::Iterator *piterIn) : piter(piterIn) { }; - ~CLevelDBIterator(); + ~CDBIterator(); bool Valid(); @@ -121,7 +121,7 @@ public: }; -class CLevelDBWrapper +class CDBWrapper { private: //! custom environment this database is using (may be NULL in case of default environment) @@ -152,8 +152,8 @@ public: * @param[in] fMemory If true, use leveldb's memory environment. * @param[in] fWipe If true, remove all existing data. */ - CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); - ~CLevelDBWrapper(); + CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); + ~CDBWrapper(); template bool Read(const K& key, V& value) const @@ -183,7 +183,7 @@ public: template bool Write(const K& key, const V& value, bool fSync = false) { - CLevelDBBatch batch; + CDBBatch batch; batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -210,12 +210,12 @@ public: template bool Erase(const K& key, bool fSync = false) { - CLevelDBBatch batch; + CDBBatch batch; batch.Erase(key); return WriteBatch(batch, fSync); } - bool WriteBatch(CLevelDBBatch& batch, bool fSync = false); + bool WriteBatch(CDBBatch& batch, bool fSync = false); // not available for LevelDB; provide for compatibility with BDB bool Flush() @@ -225,13 +225,13 @@ public: bool Sync() { - CLevelDBBatch batch; + CDBBatch batch; return WriteBatch(batch, true); } - CLevelDBIterator *NewIterator() + CDBIterator *NewIterator() { - return new CLevelDBIterator(pdb->NewIterator(iteroptions)); + return new CDBIterator(pdb->NewIterator(iteroptions)); } /** diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/leveldbwrapper_tests.cpp index 6ea1a515a..b1557622a 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/leveldbwrapper_tests.cpp @@ -25,13 +25,13 @@ bool is_null_key(const vector& key) { return isnull; } -BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) -BOOST_AUTO_TEST_CASE(leveldbwrapper) +BOOST_AUTO_TEST_CASE(dbwrapper) { { path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false); + CDBWrapper dbw(ph, (1 << 20), true, false); char key = 'k'; uint256 in = GetRandHash(); uint256 res; @@ -43,11 +43,11 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper) } // Test batch operations -BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) +BOOST_AUTO_TEST_CASE(dbwrapper_batch) { { path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false); + CDBWrapper dbw(ph, (1 << 20), true, false); char key = 'i'; uint256 in = GetRandHash(); @@ -57,7 +57,7 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CLevelDBBatch batch; + CDBBatch batch; batch.Write(key, in); batch.Write(key2, in2); @@ -78,11 +78,11 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_batch) } } -BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) +BOOST_AUTO_TEST_CASE(dbwrapper_iterator) { { path ph = temp_directory_path() / unique_path(); - CLevelDBWrapper dbw(ph, (1 << 20), true, false); + CDBWrapper dbw(ph, (1 << 20), true, false); // The two keys are intentionally chosen for ordering char key = 'j'; @@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE(leveldbwrapper_iterator) uint256 in2 = GetRandHash(); BOOST_CHECK(dbw.Write(key2, in2)); - boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); + boost::scoped_ptr it(const_cast(&dbw)->NewIterator()); // Be sure to seek past any earlier key (if it exists) it->Seek(key); diff --git a/src/txdb.cpp b/src/txdb.cpp index 59a3ba4ca..b05e27489 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -85,7 +85,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashAnchor, CAnchorsMap &mapAnchors, CNullifiersMap &mapNullifiers) { - CLevelDBBatch batch; + CDBBatch batch; size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -135,7 +135,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, return db.WriteBatch(batch); } -CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { +CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { @@ -162,7 +162,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); + boost::scoped_ptr pcursor(const_cast(&db)->NewIterator()); pcursor->Seek(DB_COINS); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -205,7 +205,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CLevelDBBatch batch; + CDBBatch batch; for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -229,7 +229,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CLevelDBBatch batch; + CDBBatch batch; for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); @@ -249,7 +249,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { bool CBlockTreeDB::LoadBlockIndexGuts() { - boost::scoped_ptr pcursor(NewIterator()); + boost::scoped_ptr pcursor(NewIterator()); pcursor->Seek(make_pair(DB_BLOCK_INDEX, uint256())); diff --git a/src/txdb.h b/src/txdb.h index ddc32fe0c..fa6c9d60e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -26,11 +26,11 @@ static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache in (MiB) static const int64_t nMinDbCache = 4; -/** CCoinsView backed by the LevelDB coin database (chainstate/) */ +/** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB : public CCoinsView { protected: - CLevelDBWrapper db; + CDBWrapper db; CCoinsViewDB(std::string dbName, size_t nCacheSize, bool fMemory = false, bool fWipe = false); public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); @@ -50,7 +50,7 @@ public: }; /** Access to the block database (blocks/index/) */ -class CBlockTreeDB : public CLevelDBWrapper +class CBlockTreeDB : public CDBWrapper { public: CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); From e3da7a572de0cdd14ac9f29c10c2eaeb73bcbe06 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 22 Oct 2015 21:33:06 -0400 Subject: [PATCH 11/15] leveldbwrapper file rename to dbwrapper.* --- src/Makefile.am | 4 ++-- src/Makefile.test.include | 2 +- src/{leveldbwrapper.cpp => dbwrapper.cpp} | 2 +- src/{leveldbwrapper.h => dbwrapper.h} | 6 +++--- src/paymentdisclosuredb.cpp | 2 +- src/test/{leveldbwrapper_tests.cpp => dbwrapper_tests.cpp} | 2 +- src/txdb.h | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) rename src/{leveldbwrapper.cpp => dbwrapper.cpp} (99%) rename src/{leveldbwrapper.h => dbwrapper.h} (98%) rename src/test/{leveldbwrapper_tests.cpp => dbwrapper_tests.cpp} (99%) diff --git a/src/Makefile.am b/src/Makefile.am index 0c866ff02..bff8102a8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -149,7 +149,7 @@ BITCOIN_CORE_H = \ init.h \ key.h \ keystore.h \ - leveldbwrapper.h \ + dbwrapper.h \ limitedmap.h \ main.h \ memusage.h \ @@ -241,7 +241,7 @@ libbitcoin_server_a_SOURCES = \ httprpc.cpp \ httpserver.cpp \ init.cpp \ - leveldbwrapper.cpp \ + dbwrapper.cpp \ main.cpp \ merkleblock.cpp \ metrics.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 02ae5ad3e..9374d71ac 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -60,7 +60,7 @@ BITCOIN_TESTS =\ test/getarg_tests.cpp \ test/hash_tests.cpp \ test/key_tests.cpp \ - test/leveldbwrapper_tests.cpp \ + test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/miner_tests.cpp \ diff --git a/src/leveldbwrapper.cpp b/src/dbwrapper.cpp similarity index 99% rename from src/leveldbwrapper.cpp rename to src/dbwrapper.cpp index 9933df7e4..91e9a9a2a 100644 --- a/src/leveldbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include "util.h" diff --git a/src/leveldbwrapper.h b/src/dbwrapper.h similarity index 98% rename from src/leveldbwrapper.h rename to src/dbwrapper.h index b7f6fa38c..ea59dd40c 100644 --- a/src/leveldbwrapper.h +++ b/src/dbwrapper.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_LEVELDBWRAPPER_H -#define BITCOIN_LEVELDBWRAPPER_H +#ifndef BITCOIN_DBWRAPPER_H +#define BITCOIN_DBWRAPPER_H #include "clientversion.h" #include "serialize.h" @@ -241,5 +241,5 @@ public: }; -#endif // BITCOIN_LEVELDBWRAPPER_H +#endif // BITCOIN_DBWRAPPER_H diff --git a/src/paymentdisclosuredb.cpp b/src/paymentdisclosuredb.cpp index ef32f2845..17302a167 100644 --- a/src/paymentdisclosuredb.cpp +++ b/src/paymentdisclosuredb.cpp @@ -5,7 +5,7 @@ #include "paymentdisclosuredb.h" #include "util.h" -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include diff --git a/src/test/leveldbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp similarity index 99% rename from src/test/leveldbwrapper_tests.cpp rename to src/test/dbwrapper_tests.cpp index b1557622a..3df1e1585 100644 --- a/src/test/leveldbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include "uint256.h" #include "random.h" #include "test/test_bitcoin.h" diff --git a/src/txdb.h b/src/txdb.h index fa6c9d60e..f96b07676 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -7,7 +7,7 @@ #define BITCOIN_TXDB_H #include "coins.h" -#include "leveldbwrapper.h" +#include "dbwrapper.h" #include #include From 809a429ecff21b9f68b94f3cb435341aba78c21e Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 20 Apr 2016 11:46:01 +0200 Subject: [PATCH 12/15] dbwrapper: Pass parent CDBWrapper into CDBBatch and CDBIterator Zcash: Makes future extensions easier. Original description (when this was introduced upstream): Pass parent wrapper directly instead of obfuscation key. This makes it possible for other databases which re-use this code to use other properties from the database. Add a namespace dbwrapper_private for private functions to be used only in dbwrapper.h/cpp and dbwrapper_tests. --- src/dbwrapper.h | 23 ++++++++++++++++------- src/test/dbwrapper_tests.cpp | 2 +- src/txdb.cpp | 6 +++--- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index ea59dd40c..5beaccbd0 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -24,15 +24,23 @@ public: void HandleError(const leveldb::Status& status); +class CDBWrapper; + /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch { friend class CDBWrapper; private: + const CDBWrapper &parent; leveldb::WriteBatch batch; public: + /** + * @param[in] parent CDBWrapper that this batch is to be submitted to + */ + CDBBatch(const CDBWrapper &parent) : parent(parent) { }; + template void Write(const K& key, const V& value) { @@ -64,15 +72,17 @@ public: class CDBIterator { private: + const CDBWrapper &parent; leveldb::Iterator *piter; public: /** + * @param[in] parent Parent CDBWrapper instance. * @param[in] piterIn The original leveldb iterator. */ - CDBIterator(leveldb::Iterator *piterIn) : - piter(piterIn) { }; + CDBIterator(const CDBWrapper &parent, leveldb::Iterator *piterIn) : + parent(parent), piter(piterIn) { }; ~CDBIterator(); bool Valid(); @@ -183,7 +193,7 @@ public: template bool Write(const K& key, const V& value, bool fSync = false) { - CDBBatch batch; + CDBBatch batch(*this); batch.Write(key, value); return WriteBatch(batch, fSync); } @@ -210,7 +220,7 @@ public: template bool Erase(const K& key, bool fSync = false) { - CDBBatch batch; + CDBBatch batch(*this); batch.Erase(key); return WriteBatch(batch, fSync); } @@ -225,20 +235,19 @@ public: bool Sync() { - CDBBatch batch; + CDBBatch batch(*this); return WriteBatch(batch, true); } CDBIterator *NewIterator() { - return new CDBIterator(pdb->NewIterator(iteroptions)); + return new CDBIterator(*this, pdb->NewIterator(iteroptions)); } /** * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 3df1e1585..496ae4be5 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -57,7 +57,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_batch) uint256 in3 = GetRandHash(); uint256 res; - CDBBatch batch; + CDBBatch batch(dbw); batch.Write(key, in); batch.Write(key2, in2); diff --git a/src/txdb.cpp b/src/txdb.cpp index b05e27489..7059dd6f8 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -85,7 +85,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashAnchor, CAnchorsMap &mapAnchors, CNullifiersMap &mapNullifiers) { - CDBBatch batch; + CDBBatch batch(db); size_t count = 0; size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { @@ -205,7 +205,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const { } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { - CDBBatch batch; + CDBBatch batch(*this); for (std::vector >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second); } @@ -229,7 +229,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { } bool CBlockTreeDB::WriteTxIndex(const std::vector >&vect) { - CDBBatch batch; + CDBBatch batch(*this); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) batch.Write(make_pair(DB_TXINDEX, it->first), it->second); return WriteBatch(batch); From 3923bcca7ced07c4a97c43ea69cbad90a504eb4e Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 20 Apr 2016 11:48:57 +0200 Subject: [PATCH 13/15] dbwrapper: Move `HandleError` to `dbwrapper_private` HandleError is implementation-specific. --- src/dbwrapper.cpp | 38 ++++++++++++++++++++----------------- src/dbwrapper.h | 14 +++++++++++--- src/paymentdisclosuredb.cpp | 6 +++--- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 91e9a9a2a..b34be5af7 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -14,20 +14,6 @@ #include #include -void HandleError(const leveldb::Status& status) -{ - if (status.ok()) - return; - LogPrintf("%s\n", status.ToString()); - if (status.IsCorruption()) - throw dbwrapper_error("Database corrupted"); - if (status.IsIOError()) - throw dbwrapper_error("Database I/O error"); - if (status.IsNotFound()) - throw dbwrapper_error("Database entry missing"); - throw dbwrapper_error("Unknown database error"); -} - static leveldb::Options GetOptions(size_t nCacheSize) { leveldb::Options options; @@ -60,13 +46,13 @@ CDBWrapper::CDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, b if (fWipe) { LogPrintf("Wiping LevelDB in %s\n", path.string()); leveldb::Status result = leveldb::DestroyDB(path.string(), options); - HandleError(result); + dbwrapper_private::HandleError(result); } TryCreateDirectory(path); LogPrintf("Opening LevelDB in %s\n", path.string()); } leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); - HandleError(status); + dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); } @@ -85,7 +71,7 @@ CDBWrapper::~CDBWrapper() bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) { leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch); - HandleError(status); + dbwrapper_private::HandleError(status); return true; } @@ -100,3 +86,21 @@ CDBIterator::~CDBIterator() { delete piter; } bool CDBIterator::Valid() { return piter->Valid(); } void CDBIterator::SeekToFirst() { piter->SeekToFirst(); } void CDBIterator::Next() { piter->Next(); } + +namespace dbwrapper_private { + +void HandleError(const leveldb::Status& status) +{ + if (status.ok()) + return; + LogPrintf("%s\n", status.ToString()); + if (status.IsCorruption()) + throw dbwrapper_error("Database corrupted"); + if (status.IsIOError()) + throw dbwrapper_error("Database I/O error"); + if (status.IsNotFound()) + throw dbwrapper_error("Database entry missing"); + throw dbwrapper_error("Unknown database error"); +} + +}; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 5beaccbd0..ca5b108ff 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -22,9 +22,17 @@ public: dbwrapper_error(const std::string& msg) : std::runtime_error(msg) {} }; +class CDBWrapper; + +/** These should be considered an implementation detail of the specific database. + */ +namespace dbwrapper_private { + +/** Handle database error by throwing dbwrapper_error exception. + */ void HandleError(const leveldb::Status& status); -class CDBWrapper; +}; /** Batch of changes queued to be written to a CDBWrapper */ class CDBBatch @@ -179,7 +187,7 @@ public: if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); @@ -212,7 +220,7 @@ public: if (status.IsNotFound()) return false; LogPrintf("LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } return true; } diff --git a/src/paymentdisclosuredb.cpp b/src/paymentdisclosuredb.cpp index 17302a167..1851e77ec 100644 --- a/src/paymentdisclosuredb.cpp +++ b/src/paymentdisclosuredb.cpp @@ -38,7 +38,7 @@ PaymentDisclosureDB::PaymentDisclosureDB(const boost::filesystem::path& dbPath) TryCreateDirectory(path); options.create_if_missing = true; leveldb::Status status = leveldb::DB::Open(options, path.string(), &db); - HandleError(status); // throws exception + dbwrapper_private::HandleError(status); // throws exception LogPrintf("PaymentDisclosure: Opened LevelDB successfully\n"); } @@ -62,7 +62,7 @@ bool PaymentDisclosureDB::Put(const PaymentDisclosureKey& key, const PaymentDisc leveldb::Slice slice(&ssValue[0], ssValue.size()); leveldb::Status status = db->Put(writeOptions, key.ToString(), slice); - HandleError(status); + dbwrapper_private::HandleError(status); return true; } @@ -80,7 +80,7 @@ bool PaymentDisclosureDB::Get(const PaymentDisclosureKey& key, PaymentDisclosure if (status.IsNotFound()) return false; LogPrintf("PaymentDisclosure: LevelDB read failure: %s\n", status.ToString()); - HandleError(status); + dbwrapper_private::HandleError(status); } try { From 56aa25684cb27bbf3a4d1b01a165814502a9d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Jan=C3=ADk?= Date: Fri, 5 Aug 2016 21:17:50 +0200 Subject: [PATCH 14/15] Do not shadow members in dbwrapper --- src/dbwrapper.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index ca5b108ff..1f3b4fc16 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -45,9 +45,9 @@ private: public: /** - * @param[in] parent CDBWrapper that this batch is to be submitted to + * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &parent) : parent(parent) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent) { }; template void Write(const K& key, const V& value) @@ -86,11 +86,11 @@ private: public: /** - * @param[in] parent Parent CDBWrapper instance. - * @param[in] piterIn The original leveldb iterator. + * @param[in] _parent Parent CDBWrapper instance. + * @param[in] _piter The original leveldb iterator. */ - CDBIterator(const CDBWrapper &parent, leveldb::Iterator *piterIn) : - parent(parent), piter(piterIn) { }; + CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) : + parent(_parent), piter(_piter) { }; ~CDBIterator(); bool Valid(); From 2d7bae5b33bd3db5b6c48fda915b941e57a9ebbd Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 3 Apr 2018 21:38:34 +0100 Subject: [PATCH 15/15] Update CBlockTreeDB::EraseBatchSync for dbwrapper refactor --- src/txdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 7059dd6f8..24ab0b4b5 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -217,7 +217,7 @@ bool CBlockTreeDB::WriteBatchSync(const std::vector& blockinfo) { - CLevelDBBatch batch; + CDBBatch batch(*this); for (std::vector::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) { batch.Erase(make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash())); }