diff --git a/src/services/history_scan_policy.cpp b/src/services/history_scan_policy.cpp index c83fab2..0818201 100644 --- a/src/services/history_scan_policy.cpp +++ b/src/services/history_scan_policy.cpp @@ -70,6 +70,15 @@ bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec) return lastFullSec > 0 && (nowSec - lastFullSec) > kSentSweepSafetyNetSec; } +int sentSweepNextBandSize(int prevSize, double prevMs) +{ + if (!(prevMs > 0.0)) return std::clamp(prevSize, kSentSweepBandMinBlocks, kSentSweepBandMaxBlocks); + double scale = static_cast(kSentSweepBandTargetMs) / prevMs; + scale = std::clamp(scale, 0.5, 4.0); // don't swing more than 2x smaller / 4x larger per step + int next = static_cast(std::lround(prevSize * scale)); + return std::clamp(next, kSentSweepBandMinBlocks, kSentSweepBandMaxBlocks); +} + long fullSentSweepRetryDelaySec(int consecutiveFailures) { if (consecutiveFailures <= 0) return 0; diff --git a/src/services/history_scan_policy.h b/src/services/history_scan_policy.h index 04c6d7a..e70c216 100644 --- a/src/services/history_scan_policy.h +++ b/src/services/history_scan_policy.h @@ -35,6 +35,14 @@ constexpr int kSentSweepLongCallBlocks = 2400; // tip - W beyond this: use constexpr long kSentSweepSafetyNetSec = 7 * 24 * 3600; // ~weekly full re-sweep: catches external wallet // changes the per-event guards can't see constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish +// Banded full sweep: cover [0, tip] in depth bands (minconf=a + filter-2 maxDepth=b) so each cs_main hold is +// short (~kSentSweepBandTargetMs), instead of one call that holds it for the whole sweep. The band grows in +// sparse depth (no sends) and shrinks in dense depth, so it stays near the target duration either way. +constexpr long kSentSweepBandTargetMs = 5000; +constexpr int kSentSweepBandInitialBlocks = 2000; +constexpr int kSentSweepBandMinBlocks = 500; +constexpr int kSentSweepBandMaxBlocks = 100000; +constexpr int kSentSweepBandOverlap = 20; // overlap adjacent bands; dedupe by txid constexpr int kTransparentPageMin = 1000; constexpr int kTransparentPageMax = 5000; constexpr int kStallCyclesBeforeGating = 1; @@ -71,6 +79,9 @@ bool sentSweepNeedsLongCall(int tip, int fromHeight); // import, an external same-seed restore) that the per-event guards miss. lastFullSec == 0 (no full sweep // recorded yet) never triggers one: the seeding full sweep records it. bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec); +// Next banded-sweep depth size, adapting the previous band's block count toward kSentSweepBandTargetMs (at +// most 2x down / 4x up per step), clamped to [min, max]. prevMs <= 0 (unmeasured) keeps the size. +int sentSweepNextBandSize(int prevSize, double prevMs); // Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h. long fullSentSweepRetryDelaySec(int consecutiveFailures); // Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout, diff --git a/src/services/network_refresh_service.cpp b/src/services/network_refresh_service.cpp index ce3c22f..f8bdbd6 100644 --- a/src/services/network_refresh_service.cpp +++ b/src/services/network_refresh_service.cpp @@ -1338,26 +1338,56 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr if (!(effectiveFull && !result.shieldedScanComplete)) { result.sentSweepRan = true; result.sentSweepFull = effectiveFull; - // Bigger margin on the first windowed sweep after a load (absorbs wallet-notifier lag at save). - const int margin = sentSweepMargin(snapshot.sentVerifyPending && !effectiveFull); - const json params = sentSweepParams(snapshot.shieldedAddresses.front(), currentBlockHeight, - effectiveFromHeight, margin); - // A watermark saved long ago makes a huge "windowed" call; give it the long timeout too. - const bool longCall = effectiveFull || sentSweepNeedsLongCall(currentBlockHeight, effectiveFromHeight); + const std::string& z = snapshot.shieldedAddresses.front(); const auto sweepStart = std::chrono::steady_clock::now(); + std::unordered_set sweepSeen; // dedupe across overlapping bands + auto takeRows = [&](const json& sent) { + if (!sent.is_array()) return; + for (const auto& sentTx : sent) { + auto txid = readOptional(sentTx, "txid"); + if (!txid || txid->empty()) continue; + if (!sweepSeen.insert(*txid).second) continue; // from the previous band's overlap + appendSentTransparentOutputs(result.transactions, *txid, sentTx); + knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction + } + }; try { - json sent = longCall - ? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec) - : rpc.call("z_listsentbyaddress", params); - if (sent.is_array()) { - for (const auto& sentTx : sent) { - auto txid = readOptional(sentTx, "txid"); - if (!txid || txid->empty()) continue; - appendSentTransparentOutputs(result.transactions, *txid, sentTx); - knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction + if (effectiveFull) { + // Banded full sweep: cover [0, tip] in adaptive depth bands (minconf=a + filter-2 + // maxDepth=b => depth [a, b]) so no single call holds the daemon's cs_main for the whole + // sweep. Newest band first (its minconf=0 includes the mempool); bands grow in sparse + // depth and shrink in dense depth to stay near the target duration. W is set only after + // every band succeeds, so a mid-sweep failure re-sweeps instead of advancing the mark. + int bandLow = 0; + int bandSize = kSentSweepBandInitialBlocks; + for (;;) { + // maxDepth = currentBlockHeight covers down to genesis (a genesis tx has depth = tip). + const int bandHigh = std::min(bandLow + bandSize, currentBlockHeight); + const json bandParams = json::array({z, bandLow, 2, bandHigh}); + const auto bandStart = std::chrono::steady_clock::now(); + takeRows(rpc.callLong("z_listsentbyaddress", bandParams, kFullSentSweepTimeoutSec)); + if (bandHigh >= currentBlockHeight) break; // reached the deepest possible depth + bandLow = bandHigh - kSentSweepBandOverlap; + bandSize = sentSweepNextBandSize(bandSize, msSince(bandStart)); } result.sentSweepOk = true; result.sentSweepHeight = currentBlockHeight; + } else { + // Windowed sweep: a single call. Bigger margin on the first one after a load (absorbs + // notifier lag); the long timeout only for a watermark saved far back. + const int margin = sentSweepMargin(snapshot.sentVerifyPending); + const json params = sentSweepParams(z, currentBlockHeight, effectiveFromHeight, margin); + const bool longCall = sentSweepNeedsLongCall(currentBlockHeight, effectiveFromHeight); + json sent = longCall + ? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec) + : rpc.call("z_listsentbyaddress", params); + if (sent.is_array()) { + takeRows(sent); + result.sentSweepOk = true; + result.sentSweepHeight = currentBlockHeight; + } + } + if (result.sentSweepOk) { // Anchor for the NEXT load's verify: A = W - steady margin, H_A = getblockhash(A). An // empty H_A (fetch failed) just means the next load can't verify and does a full sweep. const int newAnchor = sentSweepAnchor(currentBlockHeight, false); diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index 763bd4f..c9a68da 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -2161,7 +2161,7 @@ void testNetworkRefreshRpcCollectors() })); EXPECT_EQ(transactionRpc.calls[1].params, json::array({"zs-one", 0})); EXPECT_EQ(transactionRpc.calls[2].params, json::array({"zs-two", 0})); - EXPECT_EQ(transactionRpc.calls[3].params, json::array({"zs-one", 0})); + EXPECT_EQ(transactionRpc.calls[3].params, json::array({"zs-one", 0, 2, 321})); // banded full sweep [0, tip] EXPECT_EQ(transactionRpc.calls[5].params, json::array({"pending-send"})); EXPECT_EQ(transactionRpc.calls[6].params, json::array({"pending-send"})); EXPECT_EQ(transactionResult.blockHeight, 321); @@ -9273,6 +9273,11 @@ void testSentSweepPolicyHelpers() EXPECT_FALSE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec, 1000)); // exactly at the threshold EXPECT_TRUE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec + 1, 1000)); // past it -> due EXPECT_FALSE(sentSweepSafetyNetDue(1100, 1000)); // recent full sweep + EXPECT_EQ(sentSweepNextBandSize(2000, 0.0), 2000); // unmeasured -> unchanged + EXPECT_EQ(sentSweepNextBandSize(2000, 1.0), 8000); // very fast -> +4x (capped) + EXPECT_EQ(sentSweepNextBandSize(2000, 100000.0), 1000); // very slow -> -2x + EXPECT_EQ(sentSweepNextBandSize(kSentSweepBandMaxBlocks, 1.0), kSentSweepBandMaxBlocks); // clamp max + EXPECT_EQ(sentSweepNextBandSize(kSentSweepBandMinBlocks, 100000.0), kSentSweepBandMinBlocks); // clamp min auto windowed = sentSweepParams("z1", 2000, 1500, sentSweepMargin(true)); EXPECT_EQ(windowed.size(), static_cast(4)); EXPECT_EQ(windowed[3].get(), (2000 - 1500) + sentSweepMargin(true)); @@ -16260,13 +16265,14 @@ void testHistoryCollectorSweepAndEnrichment() EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull); EXPECT_EQ(result.sentSweepHeight, 110); } - { // No watermark: the whole-wallet sweep, given the long timeout. + { // No watermark: the whole-wallet sweep, banded. A small chain fits one band [0, tip], long timeout. auto snap = freshSnapshot(); MockRefreshRpc rpc; rpc.addResponse("listtransactions", json::array()); rpc.addResponse("z_listsentbyaddress", json::array()); + rpc.addResponse("getblockhash", "anchor"); auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); - EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0]")); + EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]")); // band [0, tip] EXPECT_EQ(rpc.longCalls, 1); EXPECT_EQ(rpc.lastLongTimeout, 120L); EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk); @@ -16308,9 +16314,9 @@ void testHistoryCollectorSweepAndEnrichment() rpc.addResponse("getblockhash", "anchor"); auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && result.sentSweepFull); - bool sawFull = false; + bool sawFull = false; // a mismatch forces a full (banded) sweep: the band [0, tip] for (const auto& c : rpc.calls) - if (c.method == "z_listsentbyaddress") sawFull = (c.params.dump() == std::string("[\"zs-one\",0]")); + if (c.method == "z_listsentbyaddress") sawFull = (c.params.dump() == std::string("[\"zs-one\",0,2,110]")); EXPECT_TRUE(sawFull); } { // A watermark saved long ago (tip - W > 2400): still a windowed call, but given the long timeout so @@ -16328,6 +16334,24 @@ void testHistoryCollectorSweepAndEnrichment() EXPECT_TRUE(rpc.longCalls >= 1); // the big windowed call uses callLong EXPECT_EQ(rpc.lastLongTimeout, 120L); } + { // Banded full sweep over a large chain: several short bands, newest first, covering down to the tip + // depth. Each band is a separate call (short cs_main hold) instead of one call for the whole sweep. + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-one"}; + snap.shieldedScanHeights = {{"zs-one", 10000}}; // scan complete; no watermark -> full -> banded + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + for (int b = 0; b < 16; ++b) rpc.addResponse("z_listsentbyaddress", json::array()); + rpc.addResponse("getblockhash", "anchor"); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 10000, 4); + EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk); + EXPECT_EQ(result.sentSweepHeight, 10000); + EXPECT_TRUE(rpc.longCalls >= 2); // more than one band + EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,2000]")); // first (newest) band + std::string lastBand; + for (const auto& c : rpc.calls) if (c.method == "z_listsentbyaddress") lastBand = c.params.dump(); + EXPECT_TRUE(lastBand.find(",10000]") != std::string::npos); // last band reaches the tip + } { // A failed sweep reports why (the caller's backoff), and keeps every previous row. auto snap = freshSnapshot(); snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""});