DragonX/dragonx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Auto merge of #889 - ebfull:new-imt-redux, r=ebfull

Browse Source 
Implement and integrate new Incremental Merkle Tree

This supersedes #823.

----

This is an implementation of a new incremental merkle tree with

* no memory safety issues
* a more sensible internal design
* better space efficiency (tree representation, witnessing)
* simpler API

It is intended that this tracks the behavior of the previous tree, which it does, as verified by tests. I even wrote a little circuit for testing that all the paths work.

This PR also integrates the tree into the codebase and deprecates the old tree in almost all of our code. (I left it alone in `zerocashTest` but everything else has been changed.)

This change is compatible with the testnet but you will need to clear your *local* blockchain data out since the serialized representation of the merkle tree is now different.

Closes #517, Closes #519, Closes #591, Closes #460, Closes #473
This commit is contained in:
zkbot
2016-05-03 17:33:27 +00:00
parent 4b2c67d212 26007222e8
commit 92f3c6082a
29 changed files with 1389 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/zerocash/PourInput.cpp
View File

@@ -11,7 +11,6 @@
* @copyright MIT license (see LICENSE file)
*****************************************************************************/
#include "IncrementalMerkleTree.h"
#include "PourInput.h"
namespace libzerocash {
@@ -21,27 +20,20 @@ PourInput::PourInput(int tree_depth): old_coin(), merkle_index(), path() {
this->old_coin = Coin(this->old_address.getPublicAddress(), 0);
// dummy merkle tree
IncrementalMerkleTree merkleTree(tree_depth);
ZCIncrementalMerkleTree merkleTree;
merkleTree.append(uint256(this->old_coin.getCoinCommitment().getCommitmentValue()));
// commitment from coin
std::vector<bool> commitment(ZC_CM_SIZE * 8);
convertBytesVectorToVector(this->old_coin.getCoinCommitment().getCommitmentValue(), commitment);
auto witness = merkleTree.witness();
auto merkle_path = witness.path();
// insert commitment into the merkle tree
std::vector<bool> index;
merkleTree.insertElement(commitment, index);
merkleTree.getWitness(index, this->path);
this->merkle_index = 1;
this->path = merkle_path.authentication_path;
this->merkle_index = convertVectorToInt(merkle_path.index);
}
PourInput::PourInput(Coin old_coin,
Address old_address,
size_t merkle_index,
merkle_authentication_path path) : old_coin(old_coin), merkle_index(merkle_index), path(path) {
this->old_address = old_address;
const libzcash::MerklePath &path) : old_address(old_address), old_coin(old_coin), path(path.authentication_path) {
this->merkle_index = convertVectorToInt(path.index);
};
} /* namespace libzerocash */

5
src/zerocash/PourInput.h
View File

@@ -15,6 +15,8 @@
#include "Coin.h"
#include "ZerocashParams.h"
#include "zcash/IncrementalMerkleTree.hpp"
namespace libzerocash {
class PourInput {
@@ -23,8 +25,7 @@ public:
PourInput(Coin old_coin,
Address old_address,
size_t merkle_index,
merkle_authentication_path path);
const libzcash::MerklePath& path);
Coin old_coin;
Address old_address;

199
src/zerocash/tests/merkleTest.cpp
View File

@@ -1,199 +0,0 @@
/** @file
*****************************************************************************
Test for Merkle tree.
*****************************************************************************
* @author This file is part of libzerocash, developed by the Zerocash
* project and contributors (see AUTHORS).
* @copyright MIT license (see LICENSE file)
*****************************************************************************/
#include "zerocash/IncrementalMerkleTree.h"
#include <iostream>
#include <vector>
#define BOOST_TEST_MODULE merkleTest
#include <boost/test/included/unit_test.hpp>
using namespace libzerocash;
using namespace std;
void constructNonzeroTestVector(std::vector< std::vector<bool> > &values, uint32_t size)
{
values.resize(0);
std::vector<bool> dummy;
dummy.resize(256);
dummy[0] = true;
for (uint32_t i = 0; i < size; i++)
{
values.push_back(dummy);
}
}
void constructZeroTestVector(std::vector< std::vector<bool> > &values, uint32_t size)
{
values.resize(0);
std::vector<bool> dummy;
dummy.resize(256);
for (uint32_t i = 0; i < size; i++)
{
values.push_back(dummy);
}
}
BOOST_AUTO_TEST_CASE( testRootOfTreeOfZerosIsZero ) {
IncrementalMerkleTree incTree;
std::vector< std::vector<bool> > values;
std::vector<bool> actual_root;
constructZeroTestVector(values, 2);
// Create an IncrementalMerkleTree over the values.
if (incTree.insertVector(values) == false) {
BOOST_ERROR("Could not insert into the tree.");
}
incTree.prune();
incTree.getRootValue(actual_root);
std::vector<bool> expected_root(32*8, 0);
BOOST_CHECK( expected_root == actual_root );
}
void add_values_to_reference(IncrementalMerkleTree &tree, std::vector< std::vector<bool> > &values) {
IncrementalMerkleTree newtree(20);
if (newtree.insertVector(values) == false) {
BOOST_ERROR("Could not insert into the tree.");
}
tree.setTo(newtree);
}
BOOST_AUTO_TEST_CASE( test_add_values_to_reference ) {
IncrementalMerkleTree incTree(20);
IncrementalMerkleTree incTree2(20);
std::vector< std::vector<bool> > values;
constructNonzeroTestVector(values, 2);
if (incTree.insertVector(values) == false) {
BOOST_ERROR("Could not insert into the tree.");
}
add_values_to_reference(incTree2, values);
{
std::vector<bool> root1, root2;
incTree.getRootValue(root1);
incTree2.getRootValue(root2);
BOOST_CHECK(root1 == root2);
}
}
BOOST_AUTO_TEST_CASE( testRootOfTreeOfNonZeroIsNonZero ) {
IncrementalMerkleTree incTree;
std::vector< std::vector<bool> > values;
std::vector<bool> actual_root;
constructNonzeroTestVector(values, 2);
// Create an IncrementalMerkleTree over the values.
if (incTree.insertVector(values) == false) {
BOOST_ERROR("Could not insert into the tree.");
}
incTree.prune();
incTree.getRootValue(actual_root);
std::vector<bool> expected_root(32*8, 0);
BOOST_CHECK( expected_root != actual_root );
}
BOOST_AUTO_TEST_CASE( testSerializationEdgeCase ) {
}
BOOST_AUTO_TEST_CASE( testCompactRepresentation ) {
for (uint32_t num_entries = 0; num_entries < 100; num_entries++) {
size_t test_depth = 64;
if (num_entries == 2) {
// This is a particular failure I'm testing with weird
// padding caused by this depth.
test_depth = 20;
}
std::vector< std::vector<bool> > values;
std::vector<bool> root1, root2;
IncrementalMerkleTree incTree(test_depth);
constructNonzeroTestVector(values, num_entries);
BOOST_REQUIRE( incTree.insertVector(values) );
BOOST_REQUIRE( incTree.prune() );
IncrementalMerkleTreeCompact compact = incTree.getCompactRepresentation();
BOOST_REQUIRE( compact.getTreeHeight() == test_depth );
// Calculate what the path to the next-added element should be.
std::vector<unsigned char> path_bytes(8);
std::vector<bool> path_bits;
libzerocash::convertIntToBytesVector(num_entries, path_bytes);
libzerocash::convertBytesVectorToVector(path_bytes, path_bits);
if (test_depth == 64) {
// Make sure the paths match.
BOOST_REQUIRE( compact.getHashList() == path_bits );
}
// Make sure there's a hash for every '1' bit down the path.
BOOST_REQUIRE( compact.getHashVec().size() == libzerocash::countOnes(path_bits) );
/* Test serializing and deserializing. */
std::vector<unsigned char> serializedCompact = compact.serialize();
IncrementalMerkleTreeCompact deserializedCompact = IncrementalMerkleTreeCompact::deserialize(serializedCompact);
BOOST_REQUIRE(compact.getTreeHeight() == deserializedCompact.getTreeHeight());
BOOST_REQUIRE(compact.getHashList() == deserializedCompact.getHashList());
BOOST_REQUIRE(compact.getHashVec() == deserializedCompact.getHashVec());
// Make sure 'restoring' the tree results in the same root.
IncrementalMerkleTree newTree(compact);
incTree.getRootValue(root1);
incTree.getRootValue(root2);
BOOST_REQUIRE( root1 == root2 );
}
}
BOOST_AUTO_TEST_CASE( testCompactDeserializationFailures ) {
IncrementalMerkleTree incTree(64);
std::vector< std::vector<bool> > values;
constructNonzeroTestVector(values, 5);
BOOST_REQUIRE( incTree.insertVector(values) );
BOOST_REQUIRE( incTree.prune() );
IncrementalMerkleTreeCompact compact = incTree.getCompactRepresentation();
/* Base the following tests off of this valid serialization. */
std::vector<unsigned char> serialized = compact.serialize();
/* Should fail if we truncate any number of bytes off the end. */
for (size_t trunc_len = 0; trunc_len < serialized.size(); trunc_len++) {
std::vector<unsigned char> truncated(serialized.begin(), serialized.begin() + trunc_len);
BOOST_CHECK_THROW(
IncrementalMerkleTreeCompact::deserialize(truncated),
std::out_of_range
);
}
/* Should fail if we append any number of extra bytes on the end. */
std::vector<unsigned char> extra_byte = serialized;
extra_byte.push_back(0x00);
BOOST_CHECK_THROW(
IncrementalMerkleTreeCompact::deserialize(extra_byte),
std::runtime_error
);
}

77
src/zerocash/tests/zerocashTest.cpp
View File

@@ -29,20 +29,20 @@
#include "zerocash/PourOutput.h"
#include "zerocash/utils/util.h"
#include "uint256.h"
using namespace std;
using namespace libsnark;
#define TEST_TREE_DEPTH 4
BOOST_AUTO_TEST_CASE( SaveAndLoadKeysFromFiles ) {
cout << "\nSaveAndLoadKeysFromFiles TEST\n" << endl;
cout << "Creating Params...\n" << endl;
libzerocash::timer_start("Param Generation");
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(TEST_TREE_DEPTH);
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::ZerocashParams p(
TEST_TREE_DEPTH,
INCREMENTAL_MERKLE_TREE_DEPTH,
&keypair
);
libzerocash::timer_stop("Param Generation");
@@ -72,11 +72,11 @@ BOOST_AUTO_TEST_CASE( SaveAndLoadKeysFromFiles ) {
libzerocash::timer_stop("Saving Verification Key");
libzerocash::timer_start("Loading Proving Key");
auto pk_loaded = libzerocash::ZerocashParams::LoadProvingKeyFromFile(pk_path, TEST_TREE_DEPTH);
auto pk_loaded = libzerocash::ZerocashParams::LoadProvingKeyFromFile(pk_path, INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::timer_stop("Loading Proving Key");
libzerocash::timer_start("Loading Verification Key");
auto vk_loaded = libzerocash::ZerocashParams::LoadVerificationKeyFromFile(vk_path, TEST_TREE_DEPTH);
auto vk_loaded = libzerocash::ZerocashParams::LoadVerificationKeyFromFile(vk_path, INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::timer_stop("Loading Verification Key");
cout << "Comparing Proving and Verification key.\n" << endl;
@@ -107,19 +107,19 @@ BOOST_AUTO_TEST_CASE( SaveAndLoadKeysFromFiles ) {
}
cout << "Creating Merkle Tree...\n" << endl;
libzerocash::IncrementalMerkleTree merkleTree(coinValues, TEST_TREE_DEPTH);
libzerocash::IncrementalMerkleTree merkleTree(coinValues, INCREMENTAL_MERKLE_TREE_DEPTH);
cout << "Successfully created Merkle Tree.\n" << endl;
std::vector<bool> index;
cout << "Creating Witness 1...\n" << endl;
merkle_authentication_path witness_1(TEST_TREE_DEPTH);
merkle_authentication_path witness_1(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::convertIntToVector(1, index);
merkleTree.getWitness(index, witness_1);
cout << "Successfully created Witness 1.\n" << endl;
cout << "Creating Witness 2...\n" << endl;
merkle_authentication_path witness_2(TEST_TREE_DEPTH);
merkle_authentication_path witness_2(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::convertIntToVector(3, index);
merkleTree.getWitness(index, witness_2);
cout << "Successfully created Witness 2.\n" << endl;
@@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE( SaveAndLoadKeysFromFiles ) {
BOOST_AUTO_TEST_CASE( PourInputOutputTest ) {
// dummy input
{
libzerocash::PourInput input(TEST_TREE_DEPTH);
libzerocash::PourInput input(INCREMENTAL_MERKLE_TREE_DEPTH);
BOOST_CHECK(input.old_coin.getValue() == 0);
BOOST_CHECK(input.old_address.getPublicAddress() == input.old_coin.getPublicAddress());
@@ -192,10 +192,10 @@ void test_pour(libzerocash::ZerocashParams& p,
std::vector<uint64_t> inputs, // values of the inputs (max 2)
std::vector<uint64_t> outputs) // values of the outputs (max 2)
{
using pour_input_state = std::tuple<libzerocash::Address, libzerocash::Coin, std::vector<bool>>;
using pour_input_state = std::tuple<libzerocash::Address, libzerocash::Coin, ZCIncrementalWitness>;
// Construct incremental merkle tree
libzerocash::IncrementalMerkleTree merkleTree(TEST_TREE_DEPTH);
ZCIncrementalMerkleTree merkleTree;
// Dummy sig_pk
vector<unsigned char> as(ZC_SIG_PK_SIZE, 'a');
@@ -210,33 +210,30 @@ void test_pour(libzerocash::ZerocashParams& p,
libzerocash::Coin coin(addr.getPublicAddress(), *it);
// commitment from coin
std::vector<bool> commitment(ZC_CM_SIZE * 8);
libzerocash::convertBytesVectorToVector(coin.getCoinCommitment().getCommitmentValue(), commitment);
uint256 commitment(coin.getCoinCommitment().getCommitmentValue());
// insert commitment into the merkle tree
std::vector<bool> index;
merkleTree.insertElement(commitment, index);
merkleTree.append(commitment);
// and append to any witnesses
for(vector<pour_input_state>::iterator wit = input_state.begin(); wit != input_state.end(); ++wit) {
std::get<2>(*wit).append(commitment);
}
// store the state temporarily
input_state.push_back(std::make_tuple(addr, coin, index));
input_state.push_back(std::make_tuple(addr, coin, merkleTree.witness()));
}
// compute the merkle root we will be working with
vector<unsigned char> rt(ZC_ROOT_SIZE);
{
vector<bool> root_bv(ZC_ROOT_SIZE * 8);
merkleTree.getRootValue(root_bv);
libzerocash::convertVectorToBytesVector(root_bv, rt);
}
auto rt_u = merkleTree.root();
std::vector<unsigned char> rt(rt_u.begin(), rt_u.end());
// get witnesses for all the input coins and construct the pours
for(vector<pour_input_state>::iterator it = input_state.begin(); it != input_state.end(); ++it) {
merkle_authentication_path path(TEST_TREE_DEPTH);
auto witness = std::get<2>(*it);
auto path = witness.path();
auto index = std::get<2>(*it);
merkleTree.getWitness(index, path);
pour_inputs.push_back(libzerocash::PourInput(std::get<1>(*it), std::get<0>(*it), libzerocash::convertVectorToInt(index), path));
pour_inputs.push_back(libzerocash::PourInput(std::get<1>(*it), std::get<0>(*it), path));
}
// construct dummy outputs with the given values
@@ -250,9 +247,9 @@ void test_pour(libzerocash::ZerocashParams& p,
}
BOOST_AUTO_TEST_CASE( PourVpubInTest ) {
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(TEST_TREE_DEPTH);
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::ZerocashParams p(
TEST_TREE_DEPTH,
INCREMENTAL_MERKLE_TREE_DEPTH,
&keypair
);
@@ -331,9 +328,9 @@ BOOST_AUTO_TEST_CASE( PourTxTest ) {
cout << "Creating Params...\n" << endl;
libzerocash::timer_start("Param Generation");
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(TEST_TREE_DEPTH);
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::ZerocashParams p(
TEST_TREE_DEPTH,
INCREMENTAL_MERKLE_TREE_DEPTH,
&keypair
);
libzerocash::timer_stop("Param Generation");
@@ -363,12 +360,12 @@ BOOST_AUTO_TEST_CASE( PourTxTest ) {
cout << "Creating Merkle Tree...\n" << endl;
libzerocash::timer_start("Merkle Tree");
libzerocash::IncrementalMerkleTree merkleTree(coinValues, TEST_TREE_DEPTH);
libzerocash::IncrementalMerkleTree merkleTree(coinValues, INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::timer_stop("Merkle Tree");
cout << "Successfully created Merkle Tree.\n" << endl;
merkle_authentication_path witness_1(TEST_TREE_DEPTH);
merkle_authentication_path witness_1(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::timer_start("Witness");
std::vector<bool> index;
@@ -384,7 +381,7 @@ BOOST_AUTO_TEST_CASE( PourTxTest ) {
}
cout << "\n" << endl;
merkle_authentication_path witness_2(TEST_TREE_DEPTH);
merkle_authentication_path witness_2(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::convertIntToVector(3, index);
if (merkleTree.getWitness(index, witness_2) == false) {
cout << "Could not get witness" << endl;
@@ -539,9 +536,9 @@ BOOST_AUTO_TEST_CASE( SimpleTxTest ) {
cout << "\nSIMPLE TRANSACTION TEST\n" << endl;
libzerocash::timer_start("Param Generation");
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(TEST_TREE_DEPTH);
auto keypair = libzerocash::ZerocashParams::GenerateNewKeyPair(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::ZerocashParams p(
TEST_TREE_DEPTH,
INCREMENTAL_MERKLE_TREE_DEPTH,
&keypair
);
libzerocash::timer_stop("Param Generation");
@@ -571,13 +568,13 @@ BOOST_AUTO_TEST_CASE( SimpleTxTest ) {
}
cout << "Creating Merkle Tree...\n" << endl;
libzerocash::IncrementalMerkleTree merkleTree(coinValues, TEST_TREE_DEPTH);
libzerocash::IncrementalMerkleTree merkleTree(coinValues, INCREMENTAL_MERKLE_TREE_DEPTH);
cout << "Successfully created Merkle Tree.\n" << endl;
std::vector<bool> index;
cout << "Creating Witness 1...\n" << endl;
merkle_authentication_path witness_1(TEST_TREE_DEPTH);
merkle_authentication_path witness_1(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::convertIntToVector(1, index);
if (merkleTree.getWitness(index, witness_1) == false) {
BOOST_ERROR("Could not get witness");
@@ -585,7 +582,7 @@ BOOST_AUTO_TEST_CASE( SimpleTxTest ) {
cout << "Successfully created Witness 1.\n" << endl;
cout << "Creating Witness 2...\n" << endl;
merkle_authentication_path witness_2(TEST_TREE_DEPTH);
merkle_authentication_path witness_2(INCREMENTAL_MERKLE_TREE_DEPTH);
libzerocash::convertIntToVector(3, index);
if (merkleTree.getWitness(index, witness_2) == false) {
cout << "Could not get witness" << endl;