IBD/sync speedups: parallel RandomX pre-verify, adaptive dbcache, P2P download fixes
- Parallel RandomX PoW pre-verification pool (CCheckQueue) run ahead of the serial connect; consensus-neutral (inline CheckRandomXSolution fallback still verifies anything not pre-verified). New -randomxverifythreads (default = -par). - Adaptive dbcache: default sizes the UTXO/coins cache to most of RAM and shrinks under memory pressure, always leaving a reserve free; -dbcache pins a fixed value. - P2P block download: bounded socket recv-drain loop (tlsmanager); frontier-block reassignment to break head-of-line stalls (-blockreassigntimeout); ProcessGetData serves a bounded batch of blocks per pass instead of one (fixes the serve-side one-block-per-tick throttle that caps download network-wide). - assumeutxo: dumptxoutset RPC + LoadSnapshot machinery + AssumeutxoData chainparams. - Signed bootstrap verification (util/bootstrap-dragonx.sh, util/sign-bootstrap.md). - gtest: RandomX pre-verify consensus-equivalence test + UTXO-snapshot round-trip; revived the gtest harness (Makefile.am include fix, Makefile.gtest.include). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
172
src/gtest/test_randomx_preverify.cpp
Normal file
172
src/gtest/test_randomx_preverify.cpp
Normal file
@@ -0,0 +1,172 @@
|
||||
// Copyright (c) 2024-2026 The DragonX developers
|
||||
// Distributed under the GPLv3 software license, see the accompanying
|
||||
// file COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html
|
||||
//
|
||||
// Consensus-equivalence test for the parallel RandomX pre-verification pool. The pool is purely an
|
||||
// optimization: a block's transient fRandomXVerified flag (set by CRandomXCheck on a real hash
|
||||
// match) only lets CheckBlockHeader SKIP the inline recompute. So for every block the pool's
|
||||
// outcome must equal the inline CheckRandomXSolution outcome — `(preVerified || inline) == inline`.
|
||||
// We exercise a valid solution, a corrupted solution, and confirm the pool never "succeeds" on a
|
||||
// block the inline check would reject.
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "arith_uint256.h"
|
||||
#include "chain.h"
|
||||
#include "chainparams.h"
|
||||
#include "pow.h"
|
||||
#include "primitives/block.h"
|
||||
#include "RandomX/src/randomx.h"
|
||||
#include "hush_defs.h"
|
||||
#include "util.h"
|
||||
#include <boost/thread.hpp>
|
||||
#include <memory>
|
||||
|
||||
extern int32_t HUSH_LOADINGBLOCKS;
|
||||
extern bool fCheckpointsEnabled;
|
||||
|
||||
namespace {
|
||||
// Compute the correct RandomX solution for a header using a standalone reference light VM, via the
|
||||
// SAME key + input helpers the validator uses (so the bytes/key match exactly).
|
||||
void ReferenceRandomXHash(const CBlockHeader& hdr, const std::string& key, unsigned char out[RANDOMX_HASH_SIZE])
|
||||
{
|
||||
std::vector<unsigned char> in = GetRandomXInput(hdr);
|
||||
randomx_flags flags = randomx_get_flags();
|
||||
randomx_cache* c = randomx_alloc_cache(flags);
|
||||
ASSERT_NE(c, nullptr);
|
||||
randomx_init_cache(c, key.data(), key.size());
|
||||
randomx_vm* vm = randomx_create_vm(flags, c, nullptr);
|
||||
ASSERT_NE(vm, nullptr);
|
||||
randomx_calculate_hash(vm, in.data(), in.size(), out);
|
||||
randomx_destroy_vm(vm);
|
||||
randomx_release_cache(c);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(RandomXPreVerify, ConsensusEquivalence)
|
||||
{
|
||||
// Force RandomX validation to actually run at low heights in the test harness.
|
||||
uint32_t savedAlgo = ASSETCHAINS_ALGO, savedRx = ASSETCHAINS_RANDOMX;
|
||||
int32_t savedVal = ASSETCHAINS_RANDOMX_VALIDATION, savedLoad = HUSH_LOADINGBLOCKS;
|
||||
bool savedCkpt = fCheckpointsEnabled;
|
||||
ASSETCHAINS_RANDOMX = 2; // a distinct nonzero algo id
|
||||
ASSETCHAINS_ALGO = ASSETCHAINS_RANDOMX;
|
||||
ASSETCHAINS_RANDOMX_VALIDATION = 1; // enforce from height 1
|
||||
HUSH_LOADINGBLOCKS = 0; // not in initial-load (else RandomX skipped)
|
||||
fCheckpointsEnabled = false; // avoid the below-checkpoint skip
|
||||
|
||||
const int32_t height = 10; // < interval+lag -> the chain-params initial key (no chainActive needed)
|
||||
|
||||
CBlockHeader hdr;
|
||||
hdr.nVersion = 4;
|
||||
hdr.hashPrevBlock = uint256S("0x0000000000000000000000000000000000000000000000000000000000000001");
|
||||
hdr.hashMerkleRoot = uint256S("0x0000000000000000000000000000000000000000000000000000000000000002");
|
||||
hdr.hashFinalSaplingRoot = uint256S("0x0000000000000000000000000000000000000000000000000000000000000003");
|
||||
hdr.nTime = 1700000000;
|
||||
hdr.nBits = 0x200f0f0f;
|
||||
hdr.nNonce = uint256S("0x0000000000000000000000000000000000000000000000000000000000000004");
|
||||
|
||||
std::string key = GetRandomXKey(height);
|
||||
ASSERT_FALSE(key.empty());
|
||||
|
||||
unsigned char good[RANDOMX_HASH_SIZE];
|
||||
ReferenceRandomXHash(hdr, key, good);
|
||||
|
||||
// Run the pool path synchronously on this thread (CRandomXCheck creates its own thread_local VM).
|
||||
auto poolVerifies = [&](const CBlockHeader& h) -> bool {
|
||||
RandomXValidatorPrepareKey(key); // load the shared cache with this key
|
||||
bool slot = false;
|
||||
CRandomXCheck chk(key, GetRandomXInput(h), h.nSolution.data(), &slot);
|
||||
chk();
|
||||
return slot;
|
||||
};
|
||||
|
||||
// Case 1 — valid solution: both inline and pool accept; equivalence holds.
|
||||
hdr.nSolution.assign(good, good + RANDOMX_HASH_SIZE);
|
||||
EXPECT_TRUE(CheckRandomXSolution(&hdr, height));
|
||||
EXPECT_TRUE(poolVerifies(hdr));
|
||||
EXPECT_EQ(poolVerifies(hdr) || CheckRandomXSolution(&hdr, height), CheckRandomXSolution(&hdr, height));
|
||||
|
||||
// Case 2 — corrupted solution: both reject; the pool must NOT set verified.
|
||||
{
|
||||
CBlockHeader bad = hdr;
|
||||
bad.nSolution[0] ^= 0xff;
|
||||
EXPECT_FALSE(CheckRandomXSolution(&bad, height));
|
||||
EXPECT_FALSE(poolVerifies(bad));
|
||||
EXPECT_EQ(poolVerifies(bad) || CheckRandomXSolution(&bad, height), CheckRandomXSolution(&bad, height));
|
||||
}
|
||||
|
||||
// Case 3 — a verified flag on the block lets CheckBlockHeader skip, but verified is only ever set
|
||||
// by a real hash match, so it can never mask an invalid block. (Pool returns false for the bad
|
||||
// block above, so its fRandomXVerified stays false and the inline path rejects it at connect.)
|
||||
|
||||
ASSETCHAINS_ALGO = savedAlgo; ASSETCHAINS_RANDOMX = savedRx;
|
||||
ASSETCHAINS_RANDOMX_VALIDATION = savedVal; HUSH_LOADINGBLOCKS = savedLoad;
|
||||
fCheckpointsEnabled = savedCkpt;
|
||||
}
|
||||
|
||||
// A/B: serial inline verification (single VM) vs the parallel pool (worker threads). Directly
|
||||
// measures the speedup the pool delivers. We don't care about validity here (mismatched solutions
|
||||
// still cost a full hash), only wall-clock. parallel must beat serial whenever >1 core is used.
|
||||
TEST(RandomXPreVerify, ParallelSpeedup)
|
||||
{
|
||||
uint32_t savedAlgo = ASSETCHAINS_ALGO, savedRx = ASSETCHAINS_RANDOMX;
|
||||
int32_t savedVal = ASSETCHAINS_RANDOMX_VALIDATION, savedLoad = HUSH_LOADINGBLOCKS;
|
||||
bool savedCkpt = fCheckpointsEnabled;
|
||||
ASSETCHAINS_RANDOMX = 2; ASSETCHAINS_ALGO = ASSETCHAINS_RANDOMX;
|
||||
ASSETCHAINS_RANDOMX_VALIDATION = 1; HUSH_LOADINGBLOCKS = 0; fCheckpointsEnabled = false;
|
||||
|
||||
const int32_t height = 10;
|
||||
std::string key = GetRandomXKey(height);
|
||||
ASSERT_FALSE(key.empty());
|
||||
ASSERT_TRUE(RandomXValidatorPrepareKey(key));
|
||||
|
||||
const int M = 16; // blocks to verify in the window
|
||||
std::vector<CBlockHeader> hdrs(M);
|
||||
for (int i = 0; i < M; i++) {
|
||||
hdrs[i].nVersion = 4;
|
||||
hdrs[i].nTime = 1700000000 + i;
|
||||
hdrs[i].nBits = 0x200f0f0f;
|
||||
hdrs[i].nNonce = ArithToUint256(arith_uint256(i + 1)); // distinct inputs
|
||||
hdrs[i].nSolution.assign(RANDOMX_HASH_SIZE, 0); // arbitrary; we time the hash
|
||||
}
|
||||
|
||||
// Serial baseline: inline single-VM verification (each call hashes, then mismatches -> false).
|
||||
int64_t t0 = GetTimeMicros();
|
||||
for (int i = 0; i < M; i++) CheckRandomXSolution(&hdrs[i], height);
|
||||
int64_t serialUs = GetTimeMicros() - t0;
|
||||
|
||||
// Parallel: spawn K-1 workers + the master (this thread) joining via Wait().
|
||||
int K = std::min(8, std::max(2, (int)boost::thread::hardware_concurrency()));
|
||||
boost::thread_group workers;
|
||||
for (int i = 0; i < K - 1; i++) workers.create_thread(&ThreadRandomXVerify);
|
||||
|
||||
std::unique_ptr<bool[]> slots(new bool[M]());
|
||||
std::vector<CRandomXCheck> checks;
|
||||
checks.reserve(M);
|
||||
for (int i = 0; i < M; i++)
|
||||
checks.push_back(CRandomXCheck(key, GetRandomXInput(hdrs[i]), hdrs[i].nSolution.data(), &slots[i]));
|
||||
|
||||
int64_t t1 = GetTimeMicros();
|
||||
{
|
||||
CCheckQueueControl<CRandomXCheck> control(&rxCheckQueue);
|
||||
control.Add(checks);
|
||||
control.Wait();
|
||||
}
|
||||
int64_t parallelUs = GetTimeMicros() - t1;
|
||||
|
||||
workers.interrupt_all();
|
||||
workers.join_all();
|
||||
|
||||
printf("[ RandomX A/B ] %d blocks: serial(1 VM)=%ldms, parallel(%d threads)=%ldms, speedup=%.1fx\n",
|
||||
M, (long)(serialUs / 1000), K, (long)(parallelUs / 1000),
|
||||
(double)serialUs / (double)std::max<int64_t>(1, parallelUs));
|
||||
|
||||
EXPECT_LT(parallelUs, serialUs); // parallel must be faster than serial on a multi-core box
|
||||
|
||||
ASSETCHAINS_ALGO = savedAlgo; ASSETCHAINS_RANDOMX = savedRx;
|
||||
ASSETCHAINS_RANDOMX_VALIDATION = savedVal; HUSH_LOADINGBLOCKS = savedLoad;
|
||||
fCheckpointsEnabled = savedCkpt;
|
||||
}
|
||||
Reference in New Issue
Block a user