DragonX/ObsidianDragon
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

perf(history): band the full sent-sweep so it doesn't hold cs_main for minutes

Browse Source 
A full z_listsentbyaddress holds the daemon's cs_main for the whole sweep --
~140 s on a large wallet on a daemon without the sent-history decrypt cache --
which starves block connection and makes every other RPC (getinfo, balances)
wait, so the node looks "busy". This runs the full sweep in depth bands so no
single call holds the lock for long.

- The full case now covers [0, tip] in bands: z_listsentbyaddress(z, minconf=a,
  filter-2, maxDepth=b) returns exactly depth [a, b]. Newest band first (its
  minconf=0 includes the mempool); results dedupe by txid across the overlap.
- Each band is a separate call, so cs_main is held only ~5 s at a time and the
  node stays responsive between bands (block connection isn't starved, the busy
  canary answers). Total CPU is unchanged -- just not in one long lock.
- sentSweepNextBandSize adapts the next band toward ~5 s: grows up to 4x in
  sparse depth (empty regions covered fast), shrinks to 1/2x in dense depth.
- W advances only after every band succeeds; a mid-sweep failure re-sweeps
  rather than marking a partial result. The windowed sweep is unchanged.
- Each band also fits the 120 s timeout on its own, so the seed completes even
  on a daemon where the single whole-wallet call used to time out.

This removes the ~2.5 min hang on the first-run seed, the weekly safety-net
re-sweep, and any invalidation. Multi-band + band-sizing covered by tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
DanS
2026-10-17 14:42:16 -05:00
co-authored by Claude Opus 4.8
parent b47d55a3d0
commit 9ff64b1846
4 changed files with 94 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_phase4.cpp
+29 -5
View File
@@ -2161,7 +2161,7 @@ void testNetworkRefreshRpcCollectors()
}));
EXPECT_EQ(transactionRpc.calls[1].params, json::array({"zs-one", 0}));
EXPECT_EQ(transactionRpc.calls[2].params, json::array({"zs-two", 0}));
EXPECT_EQ(transactionRpc.calls[3].params, json::array({"zs-one", 0}));
EXPECT_EQ(transactionRpc.calls[3].params, json::array({"zs-one", 0, 2, 321})); // banded full sweep [0, tip]
EXPECT_EQ(transactionRpc.calls[5].params, json::array({"pending-send"}));
EXPECT_EQ(transactionRpc.calls[6].params, json::array({"pending-send"}));
EXPECT_EQ(transactionResult.blockHeight, 321);
@@ -9273,6 +9273,11 @@ void testSentSweepPolicyHelpers()
EXPECT_FALSE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec, 1000)); // exactly at the threshold
EXPECT_TRUE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec + 1, 1000)); // past it -> due
EXPECT_FALSE(sentSweepSafetyNetDue(1100, 1000)); // recent full sweep
EXPECT_EQ(sentSweepNextBandSize(2000, 0.0), 2000); // unmeasured -> unchanged
EXPECT_EQ(sentSweepNextBandSize(2000, 1.0), 8000); // very fast -> +4x (capped)
EXPECT_EQ(sentSweepNextBandSize(2000, 100000.0), 1000); // very slow -> -2x
EXPECT_EQ(sentSweepNextBandSize(kSentSweepBandMaxBlocks, 1.0), kSentSweepBandMaxBlocks); // clamp max
EXPECT_EQ(sentSweepNextBandSize(kSentSweepBandMinBlocks, 100000.0), kSentSweepBandMinBlocks); // clamp min
auto windowed = sentSweepParams("z1", 2000, 1500, sentSweepMargin(true));
EXPECT_EQ(windowed.size(), static_cast<size_t>(4));
EXPECT_EQ(windowed[3].get<int>(), (2000 - 1500) + sentSweepMargin(true));
@@ -16260,13 +16265,14 @@ void testHistoryCollectorSweepAndEnrichment()
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
EXPECT_EQ(result.sentSweepHeight, 110);
}
{ // No watermark: the whole-wallet sweep, given the long timeout.
{ // No watermark: the whole-wallet sweep, banded. A small chain fits one band [0, tip], long timeout.
auto snap = freshSnapshot();
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
rpc.addResponse("getblockhash", "anchor");
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0]"));
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]")); // band [0, tip]
EXPECT_EQ(rpc.longCalls, 1);
EXPECT_EQ(rpc.lastLongTimeout, 120L);
EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk);
@@ -16308,9 +16314,9 @@ void testHistoryCollectorSweepAndEnrichment()
rpc.addResponse("getblockhash", "anchor");
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && result.sentSweepFull);
bool sawFull = false;
bool sawFull = false; // a mismatch forces a full (banded) sweep: the band [0, tip]
for (const auto& c : rpc.calls)
if (c.method == "z_listsentbyaddress") sawFull = (c.params.dump() == std::string("[\"zs-one\",0]"));
if (c.method == "z_listsentbyaddress") sawFull = (c.params.dump() == std::string("[\"zs-one\",0,2,110]"));
EXPECT_TRUE(sawFull);
}
{ // A watermark saved long ago (tip - W > 2400): still a windowed call, but given the long timeout so
@@ -16328,6 +16334,24 @@ void testHistoryCollectorSweepAndEnrichment()
EXPECT_TRUE(rpc.longCalls >= 1); // the big windowed call uses callLong
EXPECT_EQ(rpc.lastLongTimeout, 120L);
}
{ // Banded full sweep over a large chain: several short bands, newest first, covering down to the tip
// depth. Each band is a separate call (short cs_main hold) instead of one call for the whole sweep.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-one"};
snap.shieldedScanHeights = {{"zs-one", 10000}}; // scan complete; no watermark -> full -> banded
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
for (int b = 0; b < 16; ++b) rpc.addResponse("z_listsentbyaddress", json::array());
rpc.addResponse("getblockhash", "anchor");
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 10000, 4);
EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk);
EXPECT_EQ(result.sentSweepHeight, 10000);
EXPECT_TRUE(rpc.longCalls >= 2); // more than one band
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,2000]")); // first (newest) band
std::string lastBand;
for (const auto& c : rpc.calls) if (c.method == "z_listsentbyaddress") lastBand = c.params.dump();
EXPECT_TRUE(lastBand.find(",10000]") != std::string::npos); // last band reaches the tip
}
{ // A failed sweep reports why (the caller's backoff), and keeps every previous row.
auto snap = freshSnapshot();
snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""});