From 07437fb44783639608eecc5d272aa8523bd65ee7 Mon Sep 17 00:00:00 2001 From: DanS Date: Fri, 16 Oct 2026 17:56:30 -0500 Subject: [PATCH] perf(history): weekly safety-net full re-sweep for sent history Persisting the sent-sweep watermark means a relaunch runs a windowed sweep, which can't see a change made outside the GUI (a Console z_importviewingkey, an external same-seed restore) that adds an old send below the window. This adds a periodic full re-sweep so that case is still caught within ~7 days. - sentSweepSafetyNetDue(now, lastFull) = lastFull > 0 && now - lastFull > 7d (a wallet that has never full-swept doesn't trigger one; the seeding sweep records the time). lastFullTime / lastFullHeight ride in the sent_sweep snapshot; any successful full sweep (seed, verify-mismatch or safety-net) resets the clock. On load, a valid watermark without a clock starts it now, so even watermarks seeded before this field eventually re-sweep. - When due, the snapshot forces a full sweep for that cycle. Gated on: a watermark exists, a routine (non-syncing) cycle, not the Mining page, and not in full-sweep backoff -- so a full sweep that can't finish on a slow daemon backs off instead of retrying every cycle. Degrades gracefully: ~0.4 s on a note-cache daemon; callLong(120 s) with backoff on an old one (the deferred banding would keep each cs_main hold short so it fits there too). Helper + persistence covered by tests. Co-Authored-By: Claude Opus 4.8 --- src/app.h | 2 ++ src/app_network.cpp | 25 ++++++++++++++++++++++--- src/data/transaction_history_cache.cpp | 4 ++++ src/data/transaction_history_cache.h | 2 ++ src/services/history_scan_policy.cpp | 5 +++++ src/services/history_scan_policy.h | 7 +++++++ tests/test_phase4.cpp | 7 +++++++ 7 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/app.h b/src/app.h index 154e6ca..3635489 100644 --- a/src/app.h +++ b/src/app.h @@ -1767,6 +1767,8 @@ private: bool history_sent_verify_pending_ = false; // a loaded watermark owes an anchor verify before its next sweep int history_full_sweep_failures_ = 0; // whole-wallet sweeps failed in a row (backoff) std::time_t history_full_sweep_retry_at_ = 0; + std::time_t history_sent_last_full_time_ = 0; // last successful FULL sweep: drives the weekly safety-net + int history_sent_last_full_height_ = -1; // re-sweep (sentSweepSafetyNetDue) std::unordered_set history_enrich_backlog_; // txids found, not enriched yet double history_addr_scan_ms_ = 0.0; // per-address z_listreceivedbyaddress cost (estimate) std::size_t history_transparent_rows_ = 0; // rows the last listtransactions chain returned diff --git a/src/app_network.cpp b/src/app_network.cpp index 274579b..795461d 100644 --- a/src/app_network.cpp +++ b/src/app_network.cpp @@ -2023,6 +2023,12 @@ void App::loadTransactionHistoryCacheIfAvailable() history_full_sweep_failures_ = loaded.sentSweep.fullFailures; history_full_sweep_retry_at_ = loaded.sentSweep.fullRetryDelaySec > 0 ? std::time(nullptr) + loaded.sentSweep.fullRetryDelaySec : 0; + history_sent_last_full_time_ = loaded.sentSweep.lastFullTime; + history_sent_last_full_height_ = loaded.sentSweep.lastFullHeight; + // Start the safety-net clock if a watermark came back without one (an older cache, or one seeded before + // this field existed), so a windowed-only wallet still gets a full re-sweep ~7 days from now. + if (history_sent_last_full_time_ == 0 && loaded.sentSweep.valid()) + history_sent_last_full_time_ = std::time(nullptr); ++history_sent_epoch_; // the loaded watermark replaces any in-flight sweep's result transaction_history_cache_loaded_ = true; // What is on disk is exactly what was just loaded: the first identical refresh must not rewrite it. @@ -2109,6 +2115,8 @@ void App::storeTransactionHistoryCacheIfAvailable(data::HistoryStoreMode mode) job->sentSweep.fullRetryDelaySec = history_full_sweep_retry_at_ > nowSec ? static_cast(history_full_sweep_retry_at_ - nowSec) : 0; } + job->sentSweep.lastFullTime = history_sent_last_full_time_; + job->sentSweep.lastFullHeight = history_sent_last_full_height_; auto* cache = &transaction_history_cache_; // App member: outlives every job (see app.h) history_writer_.submit("tx-history", [cache, job]() { return cache->writeSnapshot(std::move(*job)); }); @@ -2811,8 +2819,15 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi if (!chatZ.empty() && listed(chatZ)) transactionSnapshot.priorityScanFloors[chatZ] = currentBlocks; } } - transactionSnapshot.sentScanFromHeight = history_sent_scan_height_; - transactionSnapshot.sentVerifyPending = history_sent_verify_pending_; + // Weekly safety net: on a watermarked wallet, force a full re-sweep now and then so a change made + // outside the GUI (a Console import, an external same-seed restore) that the per-event guards can't + // see is still caught within ~7 days. Not on the Mining page; needs a routine (non-syncing) cycle. + const bool sentSafetyNetDue = routineAllowed && history_sent_scan_height_ >= 0 && + current_page_ != ui::NavPage::Mining && + std::time(nullptr) >= history_full_sweep_retry_at_ && // a failed full sweep backs off, not retries + services::sentSweepSafetyNetDue(std::time(nullptr), history_sent_last_full_time_); + transactionSnapshot.sentScanFromHeight = sentSafetyNetDue ? -1 : history_sent_scan_height_; + transactionSnapshot.sentVerifyPending = history_sent_verify_pending_ && !sentSafetyNetDue; transactionSnapshot.sentVerifyAnchorHeight = history_sent_anchor_height_; transactionSnapshot.sentVerifyAnchorHash = history_sent_anchor_hash_; // A windowed sweep mid-pass too, now and then, so an untracked send (the Console, another wallet on @@ -2826,7 +2841,7 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi // and backs off after whole-wallet sweeps that failed. transactionSnapshot.skipSentSweep = (!routineAllowed && initial_history_scan_complete_) || - (history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) || + (!sentSafetyNetDue && history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) || (history_sent_scan_height_ < 0 && std::time(nullptr) < history_full_sweep_retry_at_); } else { // Light: rows now (a send, a mutation), no wallet-wide z-scans. @@ -2887,6 +2902,10 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi history_sent_verify_pending_ = false; // this sweep covered the window; nothing owed history_full_sweep_failures_ = 0; history_full_sweep_retry_at_ = 0; + if (result.sentSweepFull) { // a full sweep resets the safety-net clock + history_sent_last_full_time_ = now; + history_sent_last_full_height_ = result.sentSweepHeight; + } } else if (result.sentSweepRan && result.sentSweepFull && services::fullSentSweepFailureCounts(result.sentSweepErrKind)) { ++history_full_sweep_failures_; diff --git a/src/data/transaction_history_cache.cpp b/src/data/transaction_history_cache.cpp index bc7e5c7..9644efa 100644 --- a/src/data/transaction_history_cache.cpp +++ b/src/data/transaction_history_cache.cpp @@ -378,6 +378,8 @@ TransactionHistoryCache::LoadResult TransactionHistoryCache::load( sw.fullFailures = s.value("full_failures", 0); sw.fullRetryDelaySec = static_cast(s.value("full_retry_delay_s", std::int64_t{0})); sw.daemonId = s.value("daemon_id", std::string()); + sw.lastFullTime = s.value("last_full_time", std::int64_t{0}); + sw.lastFullHeight = s.value("last_full_height", -1); if (s.contains("backlog") && s["backlog"].is_array()) { for (const auto& txid : s["backlog"]) if (txid.is_string() && !txid.get().empty()) @@ -463,6 +465,8 @@ bool TransactionHistoryCache::writeSnapshot(SnapshotJob&& job) s["full_failures"] = sw.fullFailures; s["full_retry_delay_s"] = static_cast(sw.fullRetryDelaySec); s["daemon_id"] = sw.daemonId; + s["last_full_time"] = sw.lastFullTime; + s["last_full_height"] = sw.lastFullHeight; payload["sent_sweep"] = std::move(s); } diff --git a/src/data/transaction_history_cache.h b/src/data/transaction_history_cache.h index 2597842..62f7037 100644 --- a/src/data/transaction_history_cache.h +++ b/src/data/transaction_history_cache.h @@ -30,6 +30,8 @@ struct SentSweepState { int fullFailures = 0; long fullRetryDelaySec = 0; std::string daemonId; + std::int64_t lastFullTime = 0; // when the last successful FULL sweep finished: drives the weekly + int lastFullHeight = -1; // safety-net re-sweep that catches changes the per-event guards miss bool valid() const { return height >= 0; } }; diff --git a/src/services/history_scan_policy.cpp b/src/services/history_scan_policy.cpp index 87f3470..c83fab2 100644 --- a/src/services/history_scan_policy.cpp +++ b/src/services/history_scan_policy.cpp @@ -65,6 +65,11 @@ bool sentSweepNeedsLongCall(int tip, int fromHeight) return tip > 0 && fromHeight >= 0 && (tip - fromHeight) > kSentSweepLongCallBlocks; } +bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec) +{ + return lastFullSec > 0 && (nowSec - lastFullSec) > kSentSweepSafetyNetSec; +} + long fullSentSweepRetryDelaySec(int consecutiveFailures) { if (consecutiveFailures <= 0) return 0; diff --git a/src/services/history_scan_policy.h b/src/services/history_scan_policy.h index 39c1569..04c6d7a 100644 --- a/src/services/history_scan_policy.h +++ b/src/services/history_scan_policy.h @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,8 @@ constexpr int kSentSweepReorgMarginBlocks = 100; // > MAX_REORG_LENGTH (99), constexpr int kSentSweepFirstLoadExtraBlocks = 1000; // E: extra margin on the first sweep after a load, // to absorb wallet-notifier lag at the last save constexpr int kSentSweepLongCallBlocks = 2400; // tip - W beyond this: use callLong + count failures +constexpr long kSentSweepSafetyNetSec = 7 * 24 * 3600; // ~weekly full re-sweep: catches external wallet + // changes the per-event guards can't see constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish constexpr int kTransparentPageMin = 1000; constexpr int kTransparentPageMax = 5000; @@ -64,6 +67,10 @@ int sentSweepAnchor(int fromHeight, bool firstAfterLoad); // A watermark saved long ago makes a huge "windowed" call: past this gap, use callLong and count failures // toward the backoff, so it doesn't time out on every block while the daemon keeps working. bool sentSweepNeedsLongCall(int tip, int fromHeight); +// A watermarked wallet is due a periodic full re-sweep to catch changes made outside the GUI (a Console +// import, an external same-seed restore) that the per-event guards miss. lastFullSec == 0 (no full sweep +// recorded yet) never triggers one: the seeding full sweep records it. +bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec); // Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h. long fullSentSweepRetryDelaySec(int consecutiveFailures); // Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout, diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index 43ec78c..763bd4f 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -9220,6 +9220,7 @@ void testSentSweepPersistence() SentSweepState sw; sw.height = 500; sw.anchorHeight = 380; sw.anchorHash = "ha380"; sw.backlog = {"bt1", "bt2"}; sw.fullFailures = 2; sw.fullRetryDelaySec = 3600; sw.daemonId = "v1.3.0|t"; + sw.lastFullTime = 1699990000; sw.lastFullHeight = 420; EXPECT_TRUE(cache.replace(id, 500, "tip", txs, 1700000000, {}, sw)); auto loaded = cache.load(id, 500, "tip"); EXPECT_TRUE(loaded.loaded); @@ -9231,6 +9232,8 @@ void testSentSweepPersistence() EXPECT_EQ(loaded.sentSweep.fullFailures, 2); EXPECT_EQ(loaded.sentSweep.fullRetryDelaySec, 3600L); EXPECT_EQ(loaded.sentSweep.daemonId, std::string("v1.3.0|t")); + EXPECT_EQ(loaded.sentSweep.lastFullTime, static_cast(1699990000)); + EXPECT_EQ(loaded.sentSweep.lastFullHeight, 420); } { // no watermark passed -> reads invalid (W = -1), i.e. today's full-sweep behaviour TransactionHistoryCache cache(dbPath.string()); @@ -9266,6 +9269,10 @@ void testSentSweepPolicyHelpers() EXPECT_FALSE(sentSweepNeedsLongCall(1000, 999)); // tiny window EXPECT_FALSE(sentSweepNeedsLongCall(1000, -1)); // no watermark (full-sweep path) EXPECT_TRUE(sentSweepNeedsLongCall(10000, 10000 - kSentSweepLongCallBlocks - 1)); + EXPECT_FALSE(sentSweepSafetyNetDue(1000, 0)); // no full sweep recorded yet + EXPECT_FALSE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec, 1000)); // exactly at the threshold + EXPECT_TRUE(sentSweepSafetyNetDue(1000 + kSentSweepSafetyNetSec + 1, 1000)); // past it -> due + EXPECT_FALSE(sentSweepSafetyNetDue(1100, 1000)); // recent full sweep auto windowed = sentSweepParams("z1", 2000, 1500, sentSweepMargin(true)); EXPECT_EQ(windowed.size(), static_cast(4)); EXPECT_EQ(windowed[3].get(), (2000 - 1500) + sentSweepMargin(true));