perf(history): weekly safety-net full re-sweep for sent history
Persisting the sent-sweep watermark means a relaunch runs a windowed sweep, which can't see a change made outside the GUI (a Console z_importviewingkey, an external same-seed restore) that adds an old send below the window. This adds a periodic full re-sweep so that case is still caught within ~7 days. - sentSweepSafetyNetDue(now, lastFull) = lastFull > 0 && now - lastFull > 7d (a wallet that has never full-swept doesn't trigger one; the seeding sweep records the time). lastFullTime / lastFullHeight ride in the sent_sweep snapshot; any successful full sweep (seed, verify-mismatch or safety-net) resets the clock. On load, a valid watermark without a clock starts it now, so even watermarks seeded before this field eventually re-sweep. - When due, the snapshot forces a full sweep for that cycle. Gated on: a watermark exists, a routine (non-syncing) cycle, not the Mining page, and not in full-sweep backoff -- so a full sweep that can't finish on a slow daemon backs off instead of retrying every cycle. Degrades gracefully: ~0.4 s on a note-cache daemon; callLong(120 s) with backoff on an old one (the deferred banding would keep each cs_main hold short so it fits there too). Helper + persistence covered by tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -1767,6 +1767,8 @@ private:
|
||||
bool history_sent_verify_pending_ = false; // a loaded watermark owes an anchor verify before its next sweep
|
||||
int history_full_sweep_failures_ = 0; // whole-wallet sweeps failed in a row (backoff)
|
||||
std::time_t history_full_sweep_retry_at_ = 0;
|
||||
std::time_t history_sent_last_full_time_ = 0; // last successful FULL sweep: drives the weekly safety-net
|
||||
int history_sent_last_full_height_ = -1; // re-sweep (sentSweepSafetyNetDue)
|
||||
std::unordered_set<std::string> history_enrich_backlog_; // txids found, not enriched yet
|
||||
double history_addr_scan_ms_ = 0.0; // per-address z_listreceivedbyaddress cost (estimate)
|
||||
std::size_t history_transparent_rows_ = 0; // rows the last listtransactions chain returned
|
||||
|
||||
+22
-3
@@ -2023,6 +2023,12 @@ void App::loadTransactionHistoryCacheIfAvailable()
|
||||
history_full_sweep_failures_ = loaded.sentSweep.fullFailures;
|
||||
history_full_sweep_retry_at_ = loaded.sentSweep.fullRetryDelaySec > 0
|
||||
? std::time(nullptr) + loaded.sentSweep.fullRetryDelaySec : 0;
|
||||
history_sent_last_full_time_ = loaded.sentSweep.lastFullTime;
|
||||
history_sent_last_full_height_ = loaded.sentSweep.lastFullHeight;
|
||||
// Start the safety-net clock if a watermark came back without one (an older cache, or one seeded before
|
||||
// this field existed), so a windowed-only wallet still gets a full re-sweep ~7 days from now.
|
||||
if (history_sent_last_full_time_ == 0 && loaded.sentSweep.valid())
|
||||
history_sent_last_full_time_ = std::time(nullptr);
|
||||
++history_sent_epoch_; // the loaded watermark replaces any in-flight sweep's result
|
||||
transaction_history_cache_loaded_ = true;
|
||||
// What is on disk is exactly what was just loaded: the first identical refresh must not rewrite it.
|
||||
@@ -2109,6 +2115,8 @@ void App::storeTransactionHistoryCacheIfAvailable(data::HistoryStoreMode mode)
|
||||
job->sentSweep.fullRetryDelaySec = history_full_sweep_retry_at_ > nowSec
|
||||
? static_cast<long>(history_full_sweep_retry_at_ - nowSec) : 0;
|
||||
}
|
||||
job->sentSweep.lastFullTime = history_sent_last_full_time_;
|
||||
job->sentSweep.lastFullHeight = history_sent_last_full_height_;
|
||||
|
||||
auto* cache = &transaction_history_cache_; // App member: outlives every job (see app.h)
|
||||
history_writer_.submit("tx-history", [cache, job]() { return cache->writeSnapshot(std::move(*job)); });
|
||||
@@ -2811,8 +2819,15 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi
|
||||
if (!chatZ.empty() && listed(chatZ)) transactionSnapshot.priorityScanFloors[chatZ] = currentBlocks;
|
||||
}
|
||||
}
|
||||
transactionSnapshot.sentScanFromHeight = history_sent_scan_height_;
|
||||
transactionSnapshot.sentVerifyPending = history_sent_verify_pending_;
|
||||
// Weekly safety net: on a watermarked wallet, force a full re-sweep now and then so a change made
|
||||
// outside the GUI (a Console import, an external same-seed restore) that the per-event guards can't
|
||||
// see is still caught within ~7 days. Not on the Mining page; needs a routine (non-syncing) cycle.
|
||||
const bool sentSafetyNetDue = routineAllowed && history_sent_scan_height_ >= 0 &&
|
||||
current_page_ != ui::NavPage::Mining &&
|
||||
std::time(nullptr) >= history_full_sweep_retry_at_ && // a failed full sweep backs off, not retries
|
||||
services::sentSweepSafetyNetDue(std::time(nullptr), history_sent_last_full_time_);
|
||||
transactionSnapshot.sentScanFromHeight = sentSafetyNetDue ? -1 : history_sent_scan_height_;
|
||||
transactionSnapshot.sentVerifyPending = history_sent_verify_pending_ && !sentSafetyNetDue;
|
||||
transactionSnapshot.sentVerifyAnchorHeight = history_sent_anchor_height_;
|
||||
transactionSnapshot.sentVerifyAnchorHash = history_sent_anchor_hash_;
|
||||
// A windowed sweep mid-pass too, now and then, so an untracked send (the Console, another wallet on
|
||||
@@ -2826,7 +2841,7 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi
|
||||
// and backs off after whole-wallet sweeps that failed.
|
||||
transactionSnapshot.skipSentSweep =
|
||||
(!routineAllowed && initial_history_scan_complete_) ||
|
||||
(history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) ||
|
||||
(!sentSafetyNetDue && history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) ||
|
||||
(history_sent_scan_height_ < 0 && std::time(nullptr) < history_full_sweep_retry_at_);
|
||||
} else {
|
||||
// Light: rows now (a send, a mutation), no wallet-wide z-scans.
|
||||
@@ -2887,6 +2902,10 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi
|
||||
history_sent_verify_pending_ = false; // this sweep covered the window; nothing owed
|
||||
history_full_sweep_failures_ = 0;
|
||||
history_full_sweep_retry_at_ = 0;
|
||||
if (result.sentSweepFull) { // a full sweep resets the safety-net clock
|
||||
history_sent_last_full_time_ = now;
|
||||
history_sent_last_full_height_ = result.sentSweepHeight;
|
||||
}
|
||||
} else if (result.sentSweepRan && result.sentSweepFull &&
|
||||
services::fullSentSweepFailureCounts(result.sentSweepErrKind)) {
|
||||
++history_full_sweep_failures_;
|
||||
|
||||
@@ -378,6 +378,8 @@ TransactionHistoryCache::LoadResult TransactionHistoryCache::load(
|
||||
sw.fullFailures = s.value("full_failures", 0);
|
||||
sw.fullRetryDelaySec = static_cast<long>(s.value("full_retry_delay_s", std::int64_t{0}));
|
||||
sw.daemonId = s.value("daemon_id", std::string());
|
||||
sw.lastFullTime = s.value("last_full_time", std::int64_t{0});
|
||||
sw.lastFullHeight = s.value("last_full_height", -1);
|
||||
if (s.contains("backlog") && s["backlog"].is_array()) {
|
||||
for (const auto& txid : s["backlog"])
|
||||
if (txid.is_string() && !txid.get<std::string>().empty())
|
||||
@@ -463,6 +465,8 @@ bool TransactionHistoryCache::writeSnapshot(SnapshotJob&& job)
|
||||
s["full_failures"] = sw.fullFailures;
|
||||
s["full_retry_delay_s"] = static_cast<std::int64_t>(sw.fullRetryDelaySec);
|
||||
s["daemon_id"] = sw.daemonId;
|
||||
s["last_full_time"] = sw.lastFullTime;
|
||||
s["last_full_height"] = sw.lastFullHeight;
|
||||
payload["sent_sweep"] = std::move(s);
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ struct SentSweepState {
|
||||
int fullFailures = 0;
|
||||
long fullRetryDelaySec = 0;
|
||||
std::string daemonId;
|
||||
std::int64_t lastFullTime = 0; // when the last successful FULL sweep finished: drives the weekly
|
||||
int lastFullHeight = -1; // safety-net re-sweep that catches changes the per-event guards miss
|
||||
bool valid() const { return height >= 0; }
|
||||
};
|
||||
|
||||
|
||||
@@ -65,6 +65,11 @@ bool sentSweepNeedsLongCall(int tip, int fromHeight)
|
||||
return tip > 0 && fromHeight >= 0 && (tip - fromHeight) > kSentSweepLongCallBlocks;
|
||||
}
|
||||
|
||||
bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec)
|
||||
{
|
||||
return lastFullSec > 0 && (nowSec - lastFullSec) > kSentSweepSafetyNetSec;
|
||||
}
|
||||
|
||||
long fullSentSweepRetryDelaySec(int consecutiveFailures)
|
||||
{
|
||||
if (consecutiveFailures <= 0) return 0;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
@@ -31,6 +32,8 @@ constexpr int kSentSweepReorgMarginBlocks = 100; // > MAX_REORG_LENGTH (99),
|
||||
constexpr int kSentSweepFirstLoadExtraBlocks = 1000; // E: extra margin on the first sweep after a load,
|
||||
// to absorb wallet-notifier lag at the last save
|
||||
constexpr int kSentSweepLongCallBlocks = 2400; // tip - W beyond this: use callLong + count failures
|
||||
constexpr long kSentSweepSafetyNetSec = 7 * 24 * 3600; // ~weekly full re-sweep: catches external wallet
|
||||
// changes the per-event guards can't see
|
||||
constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish
|
||||
constexpr int kTransparentPageMin = 1000;
|
||||
constexpr int kTransparentPageMax = 5000;
|
||||
@@ -64,6 +67,10 @@ int sentSweepAnchor(int fromHeight, bool firstAfterLoad);
|
||||
// A watermark saved long ago makes a huge "windowed" call: past this gap, use callLong and count failures
|
||||
// toward the backoff, so it doesn't time out on every block while the daemon keeps working.
|
||||
bool sentSweepNeedsLongCall(int tip, int fromHeight);
|
||||
// A watermarked wallet is due a periodic full re-sweep to catch changes made outside the GUI (a Console
|
||||
// import, an external same-seed restore) that the per-event guards miss. lastFullSec == 0 (no full sweep
|
||||
// recorded yet) never triggers one: the seeding full sweep records it.
|
||||
bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec);
|
||||
// Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h.
|
||||
long fullSentSweepRetryDelaySec(int consecutiveFailures);
|
||||
// Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout,
|
||||
|
||||
@@ -9220,6 +9220,7 @@ void testSentSweepPersistence()
|
||||
SentSweepState sw;
|
||||
sw.height = 500; sw.anchorHeight = 380; sw.anchorHash = "ha380";
|
||||
sw.backlog = {"bt1", "bt2"}; sw.fullFailures = 2; sw.fullRetryDelaySec = 3600; sw.daemonId = "v1.3.0|t";
|
||||
sw.lastFullTime = 1699990000; sw.lastFullHeight = 420;
|
||||
EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000000, {}, sw));
|
||||
auto loaded = cache.load(id, 500, "tip");
|
||||
EXPECT_TRUE(loaded.loaded);
|
||||
@@ -9231,6 +9232,8 @@ void testSentSweepPersistence()
|
||||
EXPECT_EQ(loaded.sentSweep.fullFailures, 2);
|
||||
EXPECT_EQ(loaded.sentSweep.fullRetryDelaySec, 3600L);
|
||||
EXPECT_EQ(loaded.sentSweep.daemonId, std::string("v1.3.0|t"));
|
||||
EXPECT_EQ(loaded.sentSweep.lastFullTime, static_cast<std::int64_t>(1699990000));
|
||||
EXPECT_EQ(loaded.sentSweep.lastFullHeight, 420);
|
||||
}
|
||||
{ // no watermark passed -> reads invalid (W = -1), i.e. today's full-sweep behaviour
|
||||
TransactionHistoryCache cache(dbPath.string());
|
||||
@@ -9266,6 +9269,10 @@ void testSentSweepPolicyHelpers()
|
||||
EXPECT_FALSE(sentSweepNeedsLongCall(1000, 999)); // tiny window
|
||||
EXPECT_FALSE(sentSweepNeedsLongCall(1000, -1)); // no watermark (full-sweep path)
|
||||
EXPECT_TRUE(sentSweepNeedsLongCall(10000, 10000 - kSentSweepLongCallBlocks - 1));
|
||||
EXPECT_FALSE(sentSweepSafetyNetDue(1000, 0)); // no full sweep recorded yet
|
||||
EXPECT_FALSE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec, 1000)); // exactly at the threshold
|
||||
EXPECT_TRUE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec + 1, 1000)); // past it -> due
|
||||
EXPECT_FALSE(sentSweepSafetyNetDue(1100, 1000)); // recent full sweep
|
||||
auto windowed = sentSweepParams("z1", 2000, 1500, sentSweepMargin(true));
|
||||
EXPECT_EQ(windowed.size(), static_cast<size_t>(4));
|
||||
EXPECT_EQ(windowed[3].get<int>(), (2000 - 1500) + sentSweepMargin(true));
|
||||
|
||||
Reference in New Issue
Block a user