perf(history): persist the sent-sweep watermark (windowed relaunch)
z_listsentbyaddress re-walks the whole wallet when there's no usable watermark, which the daemon does under cs_main for every send -- hundreds of seconds, 400+ s and ~19 GB RSS on a large wallet. The watermark lived only in memory, so every relaunch, reconnect or daemon restart paid for a full sweep. This persists it so those cases run a bounded windowed sweep ([z, 0, 2, tip - W + margin]) instead (SENT-HISTORY-SWEEP Part 1). - The watermark (W = last swept tip), its anchor (A = W - margin) and anchor hash (H_A = getblockhash(A)), plus the enrichment backlog and the full- sweep backoff, ride in the encrypted history snapshot as an optional `sent_sweep` object (schema stays v1; a missing/inconsistent one reads as "no watermark" = today's behaviour, so older builds and payloads are safe). - Before the first windowed sweep after a load, the anchor is verified (getblockhash(A) == H_A). A match means any fork is inside the window; a mismatch (reorg deeper than the margin, different chain/datadir) or an inability to check drops the watermark and sweeps the whole wallet. - The first sweep after a load uses a bigger margin (absorbs notifier lag), and a watermark older than 2400 blocks uses the long timeout (callLong). - The backlog is persisted WITH W: a pure z->z send's txid lives only in a sweep result and would be lost if the window moved past an un-enriched one. - Watermark changes reach disk (folded into the store signature), and every hard case still forces a full sweep: reorg (anchor verify), GUI rescan / zap / import / address change, restoreOriginalWallet, and disconnect. Deferred (follow-ups, each only turns a correct-but-slow case faster, never a correctness risk): the banded full sweep for seeding on a pre-note-cache daemon, the external-import tripwire, and the weekly safety-net re-sweep. Part 2 (the daemon's decrypt cost) is already on the daemon's perf/sapling-note-cache branch. Pure helpers + cache round-trip + anchor-verify/large-window covered by tests; the getblockhash-after-sweep updates the pinned RPC sequences. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+141
-8
@@ -2157,13 +2157,13 @@ void testNetworkRefreshRpcCollectors()
|
||||
auto transactionResult = Refresh::collectTransactionRefreshResult(transactionRpc, snapshot, 321, 4);
|
||||
EXPECT_TRUE(transactionRpc.methodNames() == std::vector<std::string>({
|
||||
"listtransactions", "z_listreceivedbyaddress", "z_listreceivedbyaddress",
|
||||
"z_listsentbyaddress", "z_viewtransaction", "gettransaction"
|
||||
"z_listsentbyaddress", "getblockhash", "z_viewtransaction", "gettransaction"
|
||||
}));
|
||||
EXPECT_EQ(transactionRpc.calls[1].params, json::array({"zs-one", 0}));
|
||||
EXPECT_EQ(transactionRpc.calls[2].params, json::array({"zs-two", 0}));
|
||||
EXPECT_EQ(transactionRpc.calls[3].params, json::array({"zs-one", 0}));
|
||||
EXPECT_EQ(transactionRpc.calls[4].params, json::array({"pending-send"}));
|
||||
EXPECT_EQ(transactionRpc.calls[5].params, json::array({"pending-send"}));
|
||||
EXPECT_EQ(transactionRpc.calls[6].params, json::array({"pending-send"}));
|
||||
EXPECT_EQ(transactionResult.blockHeight, 321);
|
||||
EXPECT_EQ(transactionResult.newViewTxEntries.size(), static_cast<size_t>(1));
|
||||
EXPECT_EQ(transactionResult.newViewTxEntries.count("pending-send"), static_cast<size_t>(1));
|
||||
@@ -2201,7 +2201,7 @@ void testNetworkRefreshRpcCollectors()
|
||||
auto zzResult = Refresh::collectTransactionRefreshResult(zzRpc, zzSnapshot, 400, 4);
|
||||
EXPECT_TRUE(zzRpc.methodNames() == std::vector<std::string>({
|
||||
"listtransactions", "z_listreceivedbyaddress", "z_listsentbyaddress",
|
||||
"z_viewtransaction", "gettransaction"
|
||||
"getblockhash", "z_viewtransaction", "gettransaction"
|
||||
}));
|
||||
EXPECT_EQ(zzResult.transactions.size(), static_cast<size_t>(1));
|
||||
EXPECT_EQ(zzResult.transactions[0].txid, std::string("zz-send"));
|
||||
@@ -2490,7 +2490,7 @@ void testNetworkRefreshRpcCollectors()
|
||||
auto tolerantResult = Refresh::collectTransactionRefreshResult(tolerantRpc, tolerantSnapshot, 102, 4);
|
||||
EXPECT_EQ(tolerantResult.shieldedAddressesScanned, static_cast<size_t>(0)); // both skipped
|
||||
EXPECT_TRUE(tolerantResult.shieldedScanComplete);
|
||||
EXPECT_TRUE(tolerantRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress"}));
|
||||
EXPECT_TRUE(tolerantRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress", "getblockhash"}));
|
||||
}
|
||||
|
||||
Refresh::TransactionRefreshSnapshot recentSnapshot;
|
||||
@@ -2580,7 +2580,7 @@ void testNetworkRefreshRpcCollectors()
|
||||
auto finalShielded = Refresh::collectTransactionRefreshResult(
|
||||
finalShieldedRpc, partialShieldedSnapshot, 400, 0);
|
||||
EXPECT_TRUE(finalShieldedRpc.methodNames() == std::vector<std::string>({
|
||||
"listtransactions", "z_listreceivedbyaddress", "z_listsentbyaddress"
|
||||
"listtransactions", "z_listreceivedbyaddress", "z_listsentbyaddress", "getblockhash"
|
||||
}));
|
||||
EXPECT_EQ(finalShieldedRpc.calls[1].params, json::array({"zs-two", 0}));
|
||||
EXPECT_TRUE(finalShielded.shieldedScanComplete);
|
||||
@@ -2600,7 +2600,7 @@ void testNetworkRefreshRpcCollectors()
|
||||
cachedShieldedRpc.addResponse("z_listsentbyaddress", json::array());
|
||||
auto cachedShielded = Refresh::collectTransactionRefreshResult(
|
||||
cachedShieldedRpc, cachedShieldedSnapshot, 500, 0);
|
||||
EXPECT_TRUE(cachedShieldedRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress"}));
|
||||
EXPECT_TRUE(cachedShieldedRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress", "getblockhash"}));
|
||||
EXPECT_TRUE(cachedShielded.shieldedScanComplete);
|
||||
EXPECT_EQ(cachedShielded.shieldedAddressesScanned, static_cast<size_t>(0));
|
||||
EXPECT_EQ(cachedShielded.transactions.size(), static_cast<size_t>(1));
|
||||
@@ -9199,6 +9199,79 @@ void testTransactionHistoryCache()
|
||||
removeTree(dir);
|
||||
}
|
||||
|
||||
// Sent-sweep watermark persistence (SENT-HISTORY-SWEEP Part 1): the watermark + enrichment backlog
|
||||
// round-trip through the encrypted snapshot, and a missing / inconsistent one reads as "no watermark".
|
||||
void testSentSweepPersistence()
|
||||
{
|
||||
using dragonx::TransactionInfo;
|
||||
using dragonx::data::TransactionHistoryCache;
|
||||
using dragonx::data::SentSweepState;
|
||||
|
||||
fs::path dir = makeTempDir();
|
||||
fs::path dbPath = dir / "th.sqlite";
|
||||
std::string id = "mainnet|R|zs";
|
||||
std::string pass = "pw";
|
||||
std::vector<TransactionInfo> txs{ TransactionInfo{"tx1", "send", -1.0, 1700000000, 5, "zdst", "zsrc", "m"} };
|
||||
|
||||
{ // valid watermark round-trips, backlog + backoff included
|
||||
TransactionHistoryCache cache(dbPath.string());
|
||||
EXPECT_TRUE(cache.ensureOpen());
|
||||
EXPECT_TRUE(cache.unlockWithPassphrase(id, pass));
|
||||
SentSweepState sw;
|
||||
sw.height = 500; sw.anchorHeight = 380; sw.anchorHash = "ha380";
|
||||
sw.backlog = {"bt1", "bt2"}; sw.fullFailures = 2; sw.fullRetryDelaySec = 3600; sw.daemonId = "v1.3.0|t";
|
||||
EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000000, {}, sw));
|
||||
auto loaded = cache.load(id, 500, "tip");
|
||||
EXPECT_TRUE(loaded.loaded);
|
||||
EXPECT_TRUE(loaded.sentSweep.valid());
|
||||
EXPECT_EQ(loaded.sentSweep.height, 500);
|
||||
EXPECT_EQ(loaded.sentSweep.anchorHeight, 380);
|
||||
EXPECT_EQ(loaded.sentSweep.anchorHash, std::string("ha380"));
|
||||
EXPECT_EQ(loaded.sentSweep.backlog.size(), static_cast<size_t>(2));
|
||||
EXPECT_EQ(loaded.sentSweep.fullFailures, 2);
|
||||
EXPECT_EQ(loaded.sentSweep.fullRetryDelaySec, 3600L);
|
||||
EXPECT_EQ(loaded.sentSweep.daemonId, std::string("v1.3.0|t"));
|
||||
}
|
||||
{ // no watermark passed -> reads invalid (W = -1), i.e. today's full-sweep behaviour
|
||||
TransactionHistoryCache cache(dbPath.string());
|
||||
EXPECT_TRUE(cache.unlockWithPassphrase(id, pass));
|
||||
EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000001));
|
||||
auto loaded = cache.load(id, 500, "tip");
|
||||
EXPECT_TRUE(loaded.loaded);
|
||||
EXPECT_FALSE(loaded.sentSweep.valid());
|
||||
EXPECT_EQ(loaded.sentSweep.height, -1);
|
||||
}
|
||||
{ // inconsistent watermarks are refused (W above tip, or A > W)
|
||||
TransactionHistoryCache cache(dbPath.string());
|
||||
EXPECT_TRUE(cache.unlockWithPassphrase(id, pass));
|
||||
SentSweepState aboveTip; aboveTip.height = 9999; aboveTip.anchorHeight = 100; aboveTip.anchorHash = "h";
|
||||
EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000002, {}, aboveTip));
|
||||
EXPECT_FALSE(cache.load(id, 500, "tip").sentSweep.valid());
|
||||
SentSweepState anchorAbove; anchorAbove.height = 400; anchorAbove.anchorHeight = 450; anchorAbove.anchorHash = "h";
|
||||
EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000003, {}, anchorAbove));
|
||||
EXPECT_FALSE(cache.load(id, 500, "tip").sentSweep.valid());
|
||||
}
|
||||
removeTree(dir);
|
||||
}
|
||||
|
||||
// Sent-sweep pure policy helpers (anchor, margin, long-call threshold, windowed params).
|
||||
void testSentSweepPolicyHelpers()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
EXPECT_EQ(sentSweepMargin(false), kSentSweepReorgMarginBlocks);
|
||||
EXPECT_EQ(sentSweepMargin(true), kSentSweepReorgMarginBlocks + kSentSweepFirstLoadExtraBlocks);
|
||||
EXPECT_EQ(sentSweepAnchor(5000, false), 5000 - kSentSweepReorgMarginBlocks);
|
||||
EXPECT_EQ(sentSweepAnchor(50, false), 1); // clamped: W - margin < 1
|
||||
EXPECT_EQ(sentSweepAnchor(5000, true), 5000 - (kSentSweepReorgMarginBlocks + kSentSweepFirstLoadExtraBlocks));
|
||||
EXPECT_FALSE(sentSweepNeedsLongCall(1000, 999)); // tiny window
|
||||
EXPECT_FALSE(sentSweepNeedsLongCall(1000, -1)); // no watermark (full-sweep path)
|
||||
EXPECT_TRUE(sentSweepNeedsLongCall(10000, 10000 - kSentSweepLongCallBlocks - 1));
|
||||
auto windowed = sentSweepParams("z1", 2000, 1500, sentSweepMargin(true));
|
||||
EXPECT_EQ(windowed.size(), static_cast<size_t>(4));
|
||||
EXPECT_EQ(windowed[3].get<int>(), (2000 - 1500) + sentSweepMargin(true));
|
||||
EXPECT_EQ(sentSweepParams("z1", 2000, -1).size(), static_cast<size_t>(2)); // full sweep
|
||||
}
|
||||
|
||||
// GC: generating a new address changes the wallet-identity hash; the prior hash's snapshot
|
||||
// + salt must not accumulate forever. A save under a new identity prunes the old rows.
|
||||
void testTransactionHistoryCachePrunesOldWallets()
|
||||
@@ -16174,7 +16247,7 @@ void testHistoryCollectorSweepAndEnrichment()
|
||||
rpc.addResponse("listtransactions", json::array());
|
||||
rpc.addResponse("z_listsentbyaddress", json::array());
|
||||
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
|
||||
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress"}));
|
||||
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress", "getblockhash"}));
|
||||
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]"));
|
||||
EXPECT_EQ(rpc.longCalls, 0);
|
||||
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
|
||||
@@ -16191,6 +16264,63 @@ void testHistoryCollectorSweepAndEnrichment()
|
||||
EXPECT_EQ(rpc.lastLongTimeout, 120L);
|
||||
EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk);
|
||||
}
|
||||
{ // Restored watermark, anchor verify MATCHES: a windowed sweep (bigger first-after-load margin), and
|
||||
// the next anchor (A = W - steady margin, H_A) is fetched for the following load.
|
||||
auto snap = freshSnapshot();
|
||||
snap.sentScanFromHeight = 100;
|
||||
snap.sentVerifyPending = true;
|
||||
snap.sentVerifyAnchorHeight = 50;
|
||||
snap.sentVerifyAnchorHash = "anchor-hash";
|
||||
MockRefreshRpc rpc;
|
||||
rpc.addResponse("listtransactions", json::array());
|
||||
rpc.addResponse("getblockhash", "anchor-hash"); // verify: matches
|
||||
rpc.addResponse("z_listsentbyaddress", json::array());
|
||||
rpc.addResponse("getblockhash", "new-anchor-hash"); // post-success anchor fetch
|
||||
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
|
||||
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
|
||||
const int wantDepth = (110 - 100) + dragonx::services::sentSweepMargin(true);
|
||||
bool sawWindowed = false;
|
||||
for (const auto& c : rpc.calls)
|
||||
if (c.method == "z_listsentbyaddress")
|
||||
sawWindowed = (c.params.dump() == std::string("[\"zs-one\",0,2,") + std::to_string(wantDepth) + "]");
|
||||
EXPECT_TRUE(sawWindowed);
|
||||
EXPECT_EQ(result.sentSweepAnchorHeight, dragonx::services::sentSweepAnchor(110, false));
|
||||
EXPECT_EQ(result.sentSweepAnchorHash, std::string("new-anchor-hash"));
|
||||
}
|
||||
{ // Anchor verify MISMATCH (reorg deeper than the margin / different chain): drop the watermark and
|
||||
// sweep the whole wallet instead of a window that would miss the moved blocks.
|
||||
auto snap = freshSnapshot();
|
||||
snap.sentScanFromHeight = 100;
|
||||
snap.sentVerifyPending = true;
|
||||
snap.sentVerifyAnchorHeight = 50;
|
||||
snap.sentVerifyAnchorHash = "old-hash";
|
||||
MockRefreshRpc rpc;
|
||||
rpc.addResponse("listtransactions", json::array());
|
||||
rpc.addResponse("getblockhash", "reorged-hash"); // verify: mismatch
|
||||
rpc.addResponse("z_listsentbyaddress", json::array());
|
||||
rpc.addResponse("getblockhash", "anchor");
|
||||
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
|
||||
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && result.sentSweepFull);
|
||||
bool sawFull = false;
|
||||
for (const auto& c : rpc.calls)
|
||||
if (c.method == "z_listsentbyaddress") sawFull = (c.params.dump() == std::string("[\"zs-one\",0]"));
|
||||
EXPECT_TRUE(sawFull);
|
||||
}
|
||||
{ // A watermark saved long ago (tip - W > 2400): still a windowed call, but given the long timeout so
|
||||
// it doesn't time out on every block.
|
||||
Refresh::TransactionRefreshSnapshot snap;
|
||||
snap.shieldedAddresses = {"zs-one"};
|
||||
snap.shieldedScanHeights = {{"zs-one", 3000}}; // scan complete at the tip
|
||||
snap.sentScanFromHeight = 400; // tip - W = 2600 > 2400
|
||||
MockRefreshRpc rpc;
|
||||
rpc.addResponse("listtransactions", json::array());
|
||||
rpc.addResponse("z_listsentbyaddress", json::array());
|
||||
rpc.addResponse("getblockhash", "anchor");
|
||||
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 3000, 4);
|
||||
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
|
||||
EXPECT_TRUE(rpc.longCalls >= 1); // the big windowed call uses callLong
|
||||
EXPECT_EQ(rpc.lastLongTimeout, 120L);
|
||||
}
|
||||
{ // A failed sweep reports why (the caller's backoff), and keeps every previous row.
|
||||
auto snap = freshSnapshot();
|
||||
snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""});
|
||||
@@ -16371,7 +16501,8 @@ void testHistoryCollectorSweepAndEnrichment()
|
||||
auto result = Refresh::collectTransactionRefreshResult(rpc, midPass, 110, 4);
|
||||
EXPECT_FALSE(result.shieldedScanComplete);
|
||||
EXPECT_TRUE(result.sentSweepRan && !result.sentSweepFull);
|
||||
EXPECT_EQ(rpc.calls.back().params.dump(), std::string("[\"zs-a\",0,2,110]"));
|
||||
// The sweep is the 2nd-to-last call now (a getblockhash for the anchor follows a successful sweep).
|
||||
EXPECT_EQ(rpc.calls[rpc.calls.size() - 2].params.dump(), std::string("[\"zs-a\",0,2,110]"));
|
||||
midPass.allowMidPassSentSweep = false;
|
||||
MockRefreshRpc offRpc;
|
||||
offRpc.addResponse("listtransactions", json::array());
|
||||
@@ -19072,6 +19203,8 @@ int main()
|
||||
testBalanceAddressListModel();
|
||||
testExplorerBlockCache();
|
||||
testTransactionHistoryCache();
|
||||
testSentSweepPersistence();
|
||||
testSentSweepPolicyHelpers();
|
||||
testTransactionHistoryCachePrunesOldWallets();
|
||||
testHistorySnapshotSignature();
|
||||
testHistoryStorePolicy();
|
||||
|
||||
Reference in New Issue
Block a user