perf(history): scan shielded history in bounded passes, not every block

Every block re-ran z_listreceivedbyaddress for every z-address (O(wallet)
each, under cs_main) and a whole-wallet z_listsentbyaddress. On a 40-address
wallet that was ~80% of wall-clock. The history now advances in passes:

- an address is rescanned when it is stale by a number of blocks scaled to
  its measured cost (8 to 500), never-scanned and hinted addresses first
  (a hint: z_listunspent shows a note the history doesn't);
- z_listsentbyaddress covers only the blocks since the last good sweep
  (+100 reorg margin); the whole-wallet sweep runs once per connect, with
  a 10 min / 1 h / 6 h backoff after real failures;
- listtransactions pages are sized from last time, and an unchanged tip
  reuses the transparent rows;
- found-but-unenriched txids carry over as a backlog; only -5 is a
  definitive "not ours" for z_viewtransaction;
- between blocks, a cheap recent poll plus a one-address probe.

Duty model for a 40-address wallet at 10 s per scan: ~9% instead of ~80%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-09 11:06:34 -05:00
co-authored by Claude Opus 5.5
parent 8b9a498eee
commit 879de37d18
10 changed files with 1645 additions and 233 deletions
+647 -6
View File
@@ -27,6 +27,7 @@
#include "rpc/rpc_failure.h"
#include "rpc/rpc_busy_gate.h"
#include "resources/embedded_resources.h"
#include "services/history_scan_policy.h"
#include "services/network_refresh_service.h"
#include "services/refresh_scheduler.h"
#include "services/note_consolidation.h"
@@ -695,6 +696,15 @@ public:
responses[method].push_back({nlohmann::json(), true, {}, std::move(exception)});
}
int longCalls = 0;
long lastLongTimeout = 0;
nlohmann::json callLong(const std::string& method, const nlohmann::json& params, long timeoutSec) override
{
++longCalls;
lastLongTimeout = timeoutSec;
return call(method, params); // recorded like any call, so the pinned call sequences still hold
}
nlohmann::json call(const std::string& method, const nlohmann::json& params) override
{
calls.push_back({method, params});
@@ -2129,15 +2139,15 @@ void testNetworkRefreshRpcCollectors()
EXPECT_EQ(pagedResult.transactions.size(), static_cast<size_t>(1001));
EXPECT_EQ(pagedResult.transactions[0].txid, std::string("paged-1000"));
// shieldedScanTipTolerance: an address scanned a few blocks below the tip is re-scanned when
// strict (tolerance 0) but SKIPPED within tolerance — this is what lets a multi-block pass
// complete (and unsticks the perpetual "refreshing history" state).
// The pass floor: an address scanned below it is scanned again; without a floor (no pass running)
// only never-scanned addresses are. A pass's floor is its start height, so it completes however far
// the tip moves meanwhile (this is what unsticks the perpetual "refreshing history" state).
{
Refresh::TransactionRefreshSnapshot strictSnapshot;
strictSnapshot.shieldedAddresses = {"zs-a", "zs-b"};
strictSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
strictSnapshot.maxShieldedReceiveScans = 4;
strictSnapshot.shieldedScanTipTolerance = 0; // strict: heights 100 are stale at tip 102
strictSnapshot.shieldedScanFloorHeight = 102; // a pass that started at 102: both below it
MockRefreshRpc strictRpc;
strictRpc.addResponse("listtransactions", json::array());
strictRpc.addResponse("z_listreceivedbyaddress", json::array());
@@ -2149,8 +2159,7 @@ void testNetworkRefreshRpcCollectors()
Refresh::TransactionRefreshSnapshot tolerantSnapshot;
tolerantSnapshot.shieldedAddresses = {"zs-a", "zs-b"};
tolerantSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
tolerantSnapshot.maxShieldedReceiveScans = 4;
tolerantSnapshot.shieldedScanTipTolerance = 5; // 100 >= 102-5 -> still fresh
tolerantSnapshot.maxShieldedReceiveScans = 4; // no floor: no pass is running
MockRefreshRpc tolerantRpc;
tolerantRpc.addResponse("listtransactions", json::array());
// No z_listreceivedbyaddress responses: the addresses must be skipped (else call() throws).
@@ -2279,6 +2288,7 @@ void testNetworkRefreshRpcCollectors()
staleProgressSnapshot.shieldedAddresses = {"zs-current", "zs-stale", "zs-missing"};
staleProgressSnapshot.shieldedScanHeights = {{"zs-current", 500}, {"zs-stale", 499}};
staleProgressSnapshot.maxShieldedReceiveScans = 1;
staleProgressSnapshot.shieldedScanFloorHeight = 500;
staleProgressSnapshot.previousTransactions.push_back(dragonx::TransactionInfo{
"current-shielded", "receive", 0.75, 110, 12, "zs-current", "", ""
});
@@ -9840,6 +9850,632 @@ void testScopeScanAndLockedNotes()
EXPECT_FALSE(noteIsLocked(json::array()));
}
// C43/C53/C52: the pure history-scan policy (services/history_scan_policy).
void testHistoryScanPolicy()
{
using namespace dragonx::services;
using dragonx::rpc::RpcFailureKind;
// A routine re-pass is due after enough blocks that it costs ~10% of wall-clock, within [8, 500].
EXPECT_EQ(historyStaleAfterBlocks(0, 0.0), 8);
EXPECT_EQ(historyStaleAfterBlocks(4, 500.0), 8); // 2 s of scans: the floor
EXPECT_EQ(historyStaleAfterBlocks(30, 10000.0), 84); // 300 s / (36 s * 10%) = 83.3
EXPECT_EQ(historyStaleAfterBlocks(1000, 60000.0), 500); // capped
EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits<double>::quiet_NaN()), 8);
EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits<double>::infinity()), 500);
// The per-address cost estimate rises at once and decays 25% per sample; no sample, no change.
EXPECT_NEAR(updateScanCostEstimateMs(0.0, 400.0), 400.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(400.0, 800.0), 800.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(800.0, 400.0), 700.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(800.0, 0.0), 800.0, 1e-9);
// listtransactions' first page: last time's rows plus headroom, within [1000, 5000].
EXPECT_EQ(transparentPageSize(0), 1000);
EXPECT_EQ(transparentPageSize(900), 1100);
EXPECT_EQ(transparentPageSize(1500), 1700);
EXPECT_EQ(transparentPageSize(4800), 5000);
EXPECT_EQ(transparentPageSize(100000), 5000);
// The sent sweep: the whole wallet without a usable watermark, else a depth window. Never 3 params.
EXPECT_TRUE(sentSweepIsFull(0, 50));
EXPECT_TRUE(sentSweepIsFull(100, -1));
EXPECT_TRUE(sentSweepIsFull(100, 101)); // a watermark above the tip (reorg)
EXPECT_FALSE(sentSweepIsFull(100, 100));
EXPECT_EQ(sentSweepParams("zs-one", 100, -1).dump(), std::string("[\"zs-one\",0]"));
const auto windowed = sentSweepParams("zs-one", 110, 100);
EXPECT_EQ(windowed.dump(), std::string("[\"zs-one\",0,2,110]"));
EXPECT_TRUE(windowed.size() == 4 && windowed[1].is_number_integer() && windowed[2].is_number_integer() &&
windowed[3].is_number_integer());
// Backoff after whole-wallet sweeps that failed after the daemon did the work.
EXPECT_EQ(fullSentSweepRetryDelaySec(0), 0L);
EXPECT_EQ(fullSentSweepRetryDelaySec(1), 600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(2), 3600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(3), 21600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(9), 21600L);
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::ReplyTimeout));
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::AppError));
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::Other));
for (auto kind : {RpcFailureKind::Deferred, RpcFailureKind::Aborted, RpcFailureKind::NotListening,
RpcFailureKind::QueueFull, RpcFailureKind::Warmup}) {
EXPECT_FALSE(fullSentSweepFailureCounts(kind));
}
// The between-blocks probe takes the address scanned longest ago, never-scanned first.
EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"b", 40}, {"c", 60}}), static_cast<size_t>(1));
EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"c", 60}}), static_cast<size_t>(1));
EXPECT_EQ(oldestScannedAddressIndex({}, {}), static_cast<size_t>(0));
// decideTxCycle.
auto cycle = [](void (*setup)(TxCycleInputs&)) {
TxCycleInputs in;
in.heightKnown = true;
setup(in);
return decideTxCycle(in);
};
EXPECT_TRUE(cycle([](TxCycleInputs&) {}) == TxCycle::None);
// A new block: a Full cycle only when the throttle allows; otherwise it waits.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.gateOpen = true; }) == TxCycle::Full);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; }) == TxCycle::None);
// Explicit scan work skips the throttle; routine work waits for it.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; }) == TxCycle::Full);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; }) == TxCycle::None);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; in.gateOpen = true; }) == TxCycle::Full);
// A mutation or the first load gets rows now through a Light cycle (no wallet-wide z-scans).
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.dirty = true; }) == TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.neverFetched = true; }) == TxCycle::Light);
// Tracking a send alone needs a new tip for a Light cycle, so it can't loop them at one tip.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; in.newTip = true; }) == TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; }) == TxCycle::None);
// Nothing scans while syncing or before the tip is known, explicit work included.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.dirty = true; in.syncing = true; }) ==
TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.heightKnown = false; }) == TxCycle::None);
// The between-blocks poll probes one address only when the throttle allows.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; }) == TxCycle::Recent);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; }) == TxCycle::RecentWithProbe);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; in.syncing = true; }) ==
TxCycle::Recent);
// At a new tip the cheap poll runs only while the throttle holds the Full cycle back.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; }) == TxCycle::Recent);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; in.gateOpen = true; }) ==
TxCycle::Full);
}
namespace {
// Answers every call with an empty array and records the methods (for simulations).
class RecordingRefreshRpc : public dragonx::services::NetworkRefreshService::RefreshRpcGateway {
public:
std::vector<std::string> methods;
nlohmann::json call(const std::string& method, const nlohmann::json&) override
{
methods.push_back(method);
return nlohmann::json::array();
}
};
} // namespace
// C43 end to end: the policy, the pass and the real collector over 6 virtual hours on a large wallet (40
// z-addresses at 10 s per wallet-wide call, 36 s blocks, the Overview cadence), mirroring the App glue in
// refreshTransactionData / startTransactionCycle. The first load completes, routine passes keep
// completing, and the scans hold cs_main for at most ~10% of the time once the first load is done.
void testHistoryDutyModel()
{
using Refresh = dragonx::services::NetworkRefreshService;
using namespace dragonx::services;
constexpr int kAddresses = 40;
constexpr std::size_t kBudget = 4;
constexpr double kCallSec = 10.0;
constexpr double kTickSec = 10.0;
constexpr double kBlockSec = 36.0;
constexpr double kEndSec = 6 * 3600.0;
constexpr double kLateSec = 2 * 3600.0;
std::vector<std::string> addrs;
for (int i = 0; i < kAddresses; ++i) addrs.push_back("zs-d" + std::to_string(i));
std::unordered_map<std::string, int> heights;
ShieldedHistoryPass pass;
bool initialComplete = false;
int lastTxTip = -1, watermark = -1;
std::size_t cursor = 0;
double lastScanAt = -1.0, lastScanCost = 0.0, busyUntil = 0.0, lateScanSec = 0.0;
int routinePassesDone = 0;
for (double now = 0.0; now < kEndSec; now += kTickSec) {
if (now < busyUntil) continue; // one cycle in flight at a time
const int tip = 1000 + static_cast<int>(now / kBlockSec);
const int staleAfter = historyStaleAfterBlocks(addrs.size(), kCallSec * 1000.0);
const auto work = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete);
TxCycleInputs in;
in.neverFetched = lastTxTip < 0;
in.newTip = tip != lastTxTip;
in.explicitScanWork = work.explicitWork;
in.routineScanWork = work.routineWork;
in.gateOpen = lastScanAt < 0.0 || now - lastScanAt >= lastScanCost / kHistoryScanDuty;
in.heightKnown = true;
const TxCycle cycle = decideTxCycle(in);
if (cycle != TxCycle::Full && cycle != TxCycle::Light) continue;
const bool full = cycle == TxCycle::Full;
const bool routineAllowed = in.gateOpen;
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = addrs;
snap.shieldedScanHeights = heights;
if (full) {
const bool wasActive = pass.active();
pass.begin(tip, addrs, heights, staleAfter, initialComplete, routineAllowed);
if (!wasActive && pass.active()) cursor = 0;
snap.shieldedScanFloorHeight = pass.scanFloor(tip, routineAllowed);
snap.shieldedScanStartIndex = cursor;
snap.maxShieldedReceiveScans = kBudget;
for (const auto& a : addrs) {
if (!heights.count(a)) snap.priorityScanFloors[a] = tip;
}
snap.sentScanFromHeight = watermark;
snap.skipSentSweep = (!routineAllowed && initialComplete) || (watermark >= 0 && watermark == tip);
snap.allowMidPassSentSweep = routineAllowed && watermark >= 0 && watermark < tip &&
tip - watermark >= std::max(kHistoryMinStaleBlocks, staleAfter / 4);
} else {
snap.skipShieldedScans = true;
snap.skipSentSweep = true;
}
RecordingRefreshRpc rpc;
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 25);
double cost = 0.0, sweepCost = 0.0;
for (const auto& m : rpc.methods) {
if (m == "z_listreceivedbyaddress") cost += kCallSec;
if (m == "z_listsentbyaddress") { cost += kCallSec; sweepCost += kCallSec; }
}
busyUntil = now + cost;
if (now >= kLateSec) lateScanSec += cost;
lastTxTip = tip;
if (!full) continue;
lastScanCost = (result.sentSweepRan && result.sentSweepFull) ? std::max(0.001, cost - sweepCost) : cost;
lastScanAt = now + cost;
const bool wasActive = pass.active();
const bool wasExplicit = pass.explicitPass();
heights = result.shieldedScanHeights;
cursor = result.nextShieldedScanStartIndex;
if (result.sentSweepOk) watermark = result.sentSweepHeight;
pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded);
if (wasActive && !pass.active() && !wasExplicit) ++routinePassesDone;
const auto after = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete);
if (after.total > 0 && after.missing == 0 && !(pass.active() && pass.explicitPass())) initialComplete = true;
}
EXPECT_TRUE(initialComplete);
EXPECT_TRUE(routinePassesDone >= 3);
const double share = lateScanSec / (kEndSec - kLateSec);
EXPECT_TRUE(share <= 0.12);
EXPECT_TRUE(share >= 0.05); // and it does keep scanning (a stuck model would read 0)
}
// C43: a pass re-scans every z-address once, ending when each is at or above the pass's start height.
void testShieldedHistoryPass()
{
using namespace dragonx::services;
const std::vector<std::string> addrs = {"z1", "z2", "z3"};
ShieldedHistoryPass pass;
// First load: nothing scanned yet, all of it explicit work.
auto work = pass.evaluate(100, addrs, {}, 8, false);
EXPECT_EQ(work.total, static_cast<size_t>(3));
EXPECT_EQ(work.missing, static_cast<size_t>(3));
EXPECT_EQ(work.pending, static_cast<size_t>(3));
EXPECT_TRUE(work.explicitWork);
EXPECT_FALSE(work.routineWork);
// All scanned at 100: nothing is due until they're older than staleAfter.
std::unordered_map<std::string, int> heights{{"z1", 100}, {"z2", 100}, {"z3", 100}};
work = pass.evaluate(105, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(0));
EXPECT_FALSE(work.explicitWork);
EXPECT_FALSE(work.routineWork);
// Stale: a routine pass is due, and doesn't start without the throttle.
work = pass.evaluate(109, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(3));
EXPECT_FALSE(work.explicitWork);
EXPECT_TRUE(work.routineWork);
pass.begin(109, addrs, heights, 8, true, false);
EXPECT_FALSE(pass.active());
EXPECT_EQ(pass.scanFloor(109, false), -1);
pass.begin(109, addrs, heights, 8, true, true);
EXPECT_TRUE(pass.active());
EXPECT_FALSE(pass.explicitPass());
EXPECT_EQ(pass.startHeight(), 109);
// A routine pass scans only in cycles the throttle allows.
EXPECT_EQ(pass.scanFloor(110, true), 109);
EXPECT_EQ(pass.scanFloor(110, false), -1);
// It ends once every address is at or above its floor, however far the tip has moved.
heights["z1"] = 112;
heights["z2"] = 115;
pass.finish(115, addrs, heights, 2, 2);
EXPECT_TRUE(pass.active()); // z3 is still below 109
work = pass.evaluate(115, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(1));
EXPECT_TRUE(work.routineWork);
pass.finish(140, {}, heights, 4, 0); // a list blanked by a failed enumeration proves nothing
EXPECT_TRUE(pass.active());
heights["z3"] = 140;
pass.finish(140, addrs, heights, 1, 1);
EXPECT_FALSE(pass.active());
// Before the first load completes, a pass is explicit: it may run back to back.
ShieldedHistoryPass first;
const std::unordered_map<std::string, int> cached{{"z1", 10}, {"z2", 10}, {"z3", 10}}; // the history cache
first.begin(200, addrs, cached, 8, false, false);
EXPECT_TRUE(first.active());
EXPECT_TRUE(first.explicitPass());
EXPECT_EQ(first.scanFloor(200, false), 200);
EXPECT_TRUE(first.evaluate(201, addrs, cached, 8, false).explicitWork);
first.begin(150, addrs, cached, 8, false, false); // a reorg below the floor pulls it down
EXPECT_EQ(first.startHeight(), 150);
first.reset();
EXPECT_FALSE(first.active());
EXPECT_EQ(first.startHeight(), -1);
// Scans that keep failing: explicit work turns routine (throttled) until one succeeds.
ShieldedHistoryPass stall;
stall.finish(100, addrs, {}, 2, 0);
EXPECT_TRUE(stall.stalled());
work = stall.evaluate(100, addrs, {}, 8, false);
EXPECT_FALSE(work.explicitWork);
EXPECT_TRUE(work.routineWork);
stall.finish(100, addrs, {}, 1, 1);
EXPECT_FALSE(stall.stalled());
EXPECT_TRUE(stall.evaluate(100, addrs, {}, 8, false).explicitWork);
}
// C43 with the real collector: a pass over 10 addresses, 4 per cycle, finishes in 3 cycles although the tip
// moves 5 blocks per cycle (with the old "scanned within T of the tip" rule addresses went stale again
// mid-pass and a large wallet's pass never finished), and the sent sweep runs once, when it's done.
void testHistoryPassWithCollector()
{
using Refresh = dragonx::services::NetworkRefreshService;
using namespace dragonx::services;
using json = nlohmann::json;
std::vector<std::string> addrs;
std::unordered_map<std::string, int> heights;
for (int i = 0; i < 10; ++i) {
addrs.push_back("zs-p" + std::to_string(i));
heights[addrs.back()] = 100;
}
ShieldedHistoryPass pass;
int tip = 120;
pass.begin(tip, addrs, heights, 8, true, true);
EXPECT_TRUE(pass.active());
std::size_t cursor = 0;
std::size_t scans = 0;
int cycles = 0, sweeps = 0, sweepCycle = -1;
while (pass.active() && cycles < 10) {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = addrs;
snap.shieldedScanHeights = heights;
snap.maxShieldedReceiveScans = 4;
snap.shieldedScanFloorHeight = pass.scanFloor(tip, true);
snap.shieldedScanStartIndex = cursor;
snap.sentScanFromHeight = 100;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
for (int i = 0; i < 4; ++i) rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 0);
++cycles;
scans += result.shieldedAddressesScanned;
for (const auto& call : rpc.calls) {
if (call.method == "z_listsentbyaddress") { ++sweeps; sweepCycle = cycles; }
}
heights = result.shieldedScanHeights;
cursor = result.nextShieldedScanStartIndex;
pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded);
tip += 5;
}
EXPECT_FALSE(pass.active());
EXPECT_EQ(cycles, 3);
EXPECT_EQ(scans, static_cast<size_t>(10));
EXPECT_EQ(sweeps, 1);
EXPECT_EQ(sweepCycle, 3);
// A new address while a routine pass is active and the throttle is closed: the cycle scans only it
// (priority, floor -1), not the routine pass's addresses, and the pass stays active.
std::vector<std::string> many;
std::unordered_map<std::string, int> manyHeights;
for (int i = 0; i < 12; ++i) {
many.push_back("zs-r" + std::to_string(i));
if (i != 2) manyHeights[many.back()] = 90;
}
ShieldedHistoryPass routine;
routine.begin(100, many, manyHeights, 8, true, true);
EXPECT_TRUE(routine.active());
Refresh::TransactionRefreshSnapshot explicitOnly;
explicitOnly.shieldedAddresses = many;
explicitOnly.shieldedScanHeights = manyHeights;
explicitOnly.maxShieldedReceiveScans = 4;
explicitOnly.shieldedScanFloorHeight = routine.scanFloor(105, false);
explicitOnly.shieldedScanStartIndex = 8;
explicitOnly.priorityScanFloors = {{many[2], 105}};
explicitOnly.skipSentSweep = true;
EXPECT_EQ(explicitOnly.shieldedScanFloorHeight, -1);
MockRefreshRpc newAddrRpc;
newAddrRpc.addResponse("listtransactions", json::array());
newAddrRpc.addResponse("z_listreceivedbyaddress", json::array());
auto newAddr = Refresh::collectTransactionRefreshResult(newAddrRpc, explicitOnly, 105, 0);
EXPECT_TRUE(newAddrRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listreceivedbyaddress"}));
EXPECT_EQ(newAddrRpc.calls[1].params, json::array({many[2], 0}));
EXPECT_EQ(newAddr.shieldedAddressesScanned, static_cast<size_t>(1));
routine.finish(105, many, newAddr.shieldedScanHeights, 1, 1);
EXPECT_TRUE(routine.active());
}
// C53/C52: the sent sweep, enrichment backlog and carry-forward, the adaptive page and row reuse.
void testHistoryCollectorSweepAndEnrichment()
{
using Refresh = dragonx::services::NetworkRefreshService;
using json = nlohmann::json;
using dragonx::rpc::RpcFailureKind;
using dragonx::rpc::RpcTransportError;
using dragonx::rpc::RpcError;
auto freshSnapshot = []() {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-one"};
snap.shieldedScanHeights = {{"zs-one", 110}};
return snap;
};
auto countRows = [](const Refresh::TransactionRefreshResult& r, const std::string& txid) {
return static_cast<int>(std::count_if(r.transactions.begin(), r.transactions.end(),
[&](const dragonx::TransactionInfo& t) { return t.txid == txid; }));
};
{ // Past a watermark: a windowed sweep, an ordinary call.
auto snap = freshSnapshot();
snap.sentScanFromHeight = 100;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress"}));
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]"));
EXPECT_EQ(rpc.longCalls, 0);
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
EXPECT_EQ(result.sentSweepHeight, 110);
}
{ // No watermark: the whole-wallet sweep, given the long timeout.
auto snap = freshSnapshot();
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0]"));
EXPECT_EQ(rpc.longCalls, 1);
EXPECT_EQ(rpc.lastLongTimeout, 120L);
EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk);
}
{ // A failed sweep reports why (the caller's backoff), and keeps every previous row.
auto snap = freshSnapshot();
snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""});
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addException("z_listsentbyaddress", std::make_exception_ptr(RpcTransportError(
RpcFailureKind::ReplyTimeout, "RPC request failed: Timeout was reached", CURLE_OPERATION_TIMEDOUT, 0, true)));
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(result.sentSweepRan && !result.sentSweepOk);
EXPECT_TRUE(result.sentSweepErrKind == RpcFailureKind::ReplyTimeout);
EXPECT_EQ(countRows(result, "r-old"), 1);
}
{ // skipSentSweep: no sweep at all.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_FALSE(result.sentSweepRan);
}
{ // A Light cycle: no receive scans, no sweep; the shielded rows on screen are carried.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a"};
snap.skipShieldedScans = true;
snap.skipSentSweep = true;
snap.previousTransactions.push_back(dragonx::TransactionInfo{"s-old", "receive", 2.0, 60, 4, "zs-a", "", ""});
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_EQ(countRows(result, "s-old"), 1);
}
{ // Priority addresses are scanned before the cursor's.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a", "zs-b", "zs-c", "zs-d"};
snap.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}, {"zs-c", 100}, {"zs-d", 100}};
snap.shieldedScanFloorHeight = 105;
snap.priorityScanFloors = {{"zs-d", 105}};
snap.maxShieldedReceiveScans = 2;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
Refresh::collectTransactionRefreshResult(rpc, snap, 105, 4);
EXPECT_EQ(rpc.calls[1].params, json::array({"zs-d", 0}));
EXPECT_EQ(rpc.calls[2].params, json::array({"zs-a", 0}));
}
{ // Enrichment order: this cycle's discoveries, then the backlog; past the cap they're handed back.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.enrichBacklogTxids = {"b1"};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array({
json{{"txid", "t1"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 1.0}, {"time", 70}, {"confirmations", 0}},
json{{"txid", "t2"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 2.0}, {"time", 71}, {"confirmations", 0}}
}));
rpc.addResponse("z_viewtransaction", json{{"spends", json::array()}, {"outputs", json::array()}});
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 1);
EXPECT_EQ(rpc.calls[1].params, json::array({"t1"}));
EXPECT_EQ(result.unenrichedTxids.size(), static_cast<size_t>(2));
EXPECT_EQ(result.unenrichedTxids.count("t2"), static_cast<size_t>(1));
EXPECT_EQ(result.unenrichedTxids.count("b1"), static_cast<size_t>(1));
}
// A shielded send row from the history cache (an earlier session) with no cached view: re-enriched.
// The cycles below are full rebuilds (every address scanned, the pass complete), which drop rows that
// aren't rebuilt, except the "pinned" ones still waiting for their view.
auto rebuildSnapshot = [](std::vector<dragonx::TransactionInfo> previous) {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-mine"};
snap.shieldedScanHeights = {{"zs-mine", 100}};
snap.shieldedScanFloorHeight = 110;
snap.skipSentSweep = true;
snap.previousTransactions = std::move(previous);
return snap;
};
const dragonx::TransactionInfo oldSend{"zz-old", "send", -1.5, 900, 30, "zs-peer", "zs-mine", ""};
{ // The daemon answers: the row is rebuilt from the view, dated from the settled stored row (no
// gettransaction per row).
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_viewtransaction", json{
{"spends", json::array({json{{"address", "zs-mine"}}})},
{"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.5}}})}});
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({
"listtransactions", "z_listreceivedbyaddress", "z_viewtransaction"}));
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-old"), 1);
for (const auto& t : result.transactions) {
if (t.txid != "zz-old") continue;
EXPECT_EQ(t.timestamp, static_cast<std::int64_t>(900));
EXPECT_EQ(t.confirmations, 30);
}
}
{ // A stored unsettled send still asks gettransaction; if that fails, the stored row stays (pinned),
// no undated view is cached, and it is retried.
const dragonx::TransactionInfo pendingSend{"zz-pending", "send", -1.0, 950, 0, "zs-peer", "zs-mine", ""};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_viewtransaction", json{
{"spends", json::array({json{{"address", "zs-mine"}}})},
{"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.0}}})}});
rpc.addFailure("gettransaction", "busy");
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({pendingSend}), 110, 4);
EXPECT_EQ(rpc.calls.back().method, std::string("gettransaction"));
EXPECT_EQ(result.newViewTxEntries.count("zz-pending"), static_cast<size_t>(0));
EXPECT_EQ(result.unenrichedTxids.count("zz-pending"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-pending"), 1);
for (const auto& t : result.transactions) {
if (t.txid == "zz-pending") EXPECT_EQ(t.timestamp, static_cast<std::int64_t>(950));
}
}
{ // Over the cap: still pinned, so the stored row survives the rebuild.
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 0);
EXPECT_EQ(countRows(result, "zz-old"), 1);
EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast<size_t>(1));
}
{ // Only "not a wallet tx" (-5) is definitive: cached empty, and the row is let go.
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-5, "Invalid or non-wallet transaction id")));
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(1));
EXPECT_TRUE(result.newViewTxEntries.at("zz-old").outgoing_outputs.empty());
EXPECT_EQ(countRows(result, "zz-old"), 0);
}
for (int plainError = 0; plainError < 2; ++plainError) { // anything else: retried, the row kept
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
if (plainError) rpc.addFailure("z_viewtransaction", "boom");
else rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-1, "something else")));
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(0));
EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-old"), 1);
}
{ // A pinned send to two recipients keeps both rows (not just the first per txid), in a full rebuild...
const dragonx::TransactionInfo legA{"zz-two", "send", -1.0, 900, 30, "zs-peer-a", "zs-mine", ""};
const dragonx::TransactionInfo legB{"zz-two", "send", -2.0, 900, 30, "zs-peer-b", "zs-mine", ""};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({legA, legB}), 110, 0);
EXPECT_EQ(countRows(result, "zz-two"), 2);
// ...and in a cycle that carries rows (here a Light cycle).
Refresh::TransactionRefreshSnapshot light;
light.shieldedAddresses = {"zs-mine"};
light.shieldedScanHeights = {{"zs-mine", 100}};
light.skipShieldedScans = true;
light.skipSentSweep = true;
light.previousTransactions = {legA, legB};
MockRefreshRpc lightRpc;
lightRpc.addResponse("listtransactions", json::array());
auto carried = Refresh::collectTransactionRefreshResult(lightRpc, light, 110, 0);
EXPECT_EQ(countRows(carried, "zz-two"), 2);
EXPECT_EQ(carried.unenrichedTxids.count("zz-two"), static_cast<size_t>(1));
}
{ // A windowed sweep mid-pass when allowed (rate-limited by the caller); never the whole-wallet one.
Refresh::TransactionRefreshSnapshot midPass;
midPass.shieldedAddresses = {"zs-a", "zs-b"};
midPass.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
midPass.shieldedScanFloorHeight = 110;
midPass.maxShieldedReceiveScans = 1;
midPass.sentScanFromHeight = 100;
midPass.allowMidPassSentSweep = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, midPass, 110, 4);
EXPECT_FALSE(result.shieldedScanComplete);
EXPECT_TRUE(result.sentSweepRan && !result.sentSweepFull);
EXPECT_EQ(rpc.calls.back().params.dump(), std::string("[\"zs-a\",0,2,110]"));
midPass.allowMidPassSentSweep = false;
MockRefreshRpc offRpc;
offRpc.addResponse("listtransactions", json::array());
offRpc.addResponse("z_listreceivedbyaddress", json::array());
EXPECT_FALSE(Refresh::collectTransactionRefreshResult(offRpc, midPass, 110, 4).sentSweepRan);
midPass.allowMidPassSentSweep = true;
midPass.sentScanFromHeight = -1; // no watermark: the whole-wallet sweep waits for the pass
MockRefreshRpc fullRpc;
fullRpc.addResponse("listtransactions", json::array());
fullRpc.addResponse("z_listreceivedbyaddress", json::array());
EXPECT_FALSE(Refresh::collectTransactionRefreshResult(fullRpc, midPass, 110, 4).sentSweepRan);
}
{ // C52: the first page is sized from last time's rows.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.transparentRowHint = 1500;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[0].params, json::array({"", 1700, 0}));
EXPECT_TRUE(result.transparentChainOk);
}
{ // Same tip, nothing changed: the transparent rows are reused, no listtransactions at all; a tracked
// send is rebuilt once (not reused and placeheld twice).
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.reuseTransparentRows = true;
snap.sendTxids = {"t-sent"};
snap.previousTransactions = {
dragonx::TransactionInfo{"r1", "receive", 1.0, 80, 5, "R-one", "", ""},
dragonx::TransactionInfo{"s1", "receive", 2.0, 81, 5, "zs-one", "", ""},
dragonx::TransactionInfo{"t-sent", "send", -0.5, 82, 0, "R-two", "R-one", ""},
};
MockRefreshRpc rpc;
rpc.addResponse("z_viewtransaction", json());
rpc.addResponse("gettransaction", json{{"time", 82}, {"confirmations", 0}});
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[0].method, std::string("z_viewtransaction"));
EXPECT_TRUE(result.transparentRowsReused);
EXPECT_EQ(countRows(result, "r1"), 1);
EXPECT_EQ(countRows(result, "s1"), 1);
EXPECT_EQ(countRows(result, "t-sent"), 1);
}
{ // The between-blocks poll without its probe: listtransactions only.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a", "zs-b"};
snap.maxShieldedReceiveScans = 1;
snap.skipShieldedScans = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto recent = Refresh::collectRecentTransactionRefreshResult(rpc, snap, 110);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_EQ(recent.shieldedAddressesScanned, static_cast<size_t>(0));
}
}
int main()
{
testConnectionConfig();
@@ -9863,6 +10499,11 @@ int main()
testParseMoneyZat();
testSpendableBalanceNeeded();
testOperationFilterParams();
testHistoryScanPolicy();
testShieldedHistoryPass();
testHistoryPassWithCollector();
testHistoryCollectorSweepAndEnrichment();
testHistoryDutyModel();
testNetworkRefreshResultModels();
testTrackedSendRetirement();
testBalanceDisplayLerp();