perf(history): scan shielded history in bounded passes, not every block

Every block re-ran z_listreceivedbyaddress for every z-address (O(wallet) each, under cs_main) and a whole-wallet z_listsentbyaddress. On a 40-address wallet that was ~80% of wall-clock. The history now advances in passes: - an address is rescanned when it is stale by a number of blocks scaled to its measured cost (8 to 500), never-scanned and hinted addresses first (a hint: z_listunspent shows a note the history doesn't); - z_listsentbyaddress covers only the blocks since the last good sweep (+100 reorg margin); the whole-wallet sweep runs once per connect, with a 10 min / 1 h / 6 h backoff after real failures; - listtransactions pages are sized from last time, and an unchanged tip reuses the transparent rows; - found-but-unenriched txids carry over as a backlog; only -5 is a definitive "not ours" for z_viewtransaction; - between blocks, a cheap recent poll plus a one-address probe. Duty model for a 40-address wallet at 10 s per scan: ~9% instead of ~80%. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 11:06:34 -05:00
parent 8b9a498eee
commit 879de37d18
10 changed files with 1645 additions and 233 deletions
@@ -0,0 +1,180 @@
+// DragonX Wallet - ImGui Edition
+// Copyright 2024-2026 The Hush Developers
+// Released under the GPLv3
+
+#include "history_scan_policy.h"
+
+#include <algorithm>
+#include <cmath>
+
+namespace dragonx {
+namespace services {
+
+int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs)
+{
+    if (addressCount == 0 || !(perAddressScanMs > 0.0)) return kHistoryMinStaleBlocks;
+    const double blocks = std::ceil((static_cast<double>(addressCount) * perAddressScanMs / 1000.0) /
+                                    (kHistoryBlockSeconds * kHistoryScanDuty));
+    if (!(blocks < kHistoryMaxStaleBlocks)) return kHistoryMaxStaleBlocks;   // also NaN / inf
+    return std::max(kHistoryMinStaleBlocks, static_cast<int>(blocks));
+}
+
+double updateScanCostEstimateMs(double previousMs, double sampleMs)
+{
+    if (!(sampleMs > 0.0)) return previousMs;
+    if (!(previousMs > 0.0) || sampleMs > previousMs) return sampleMs;
+    return 0.75 * previousMs + 0.25 * sampleMs;
+}
+
+int transparentPageSize(std::size_t previousRowCount)
+{
+    const std::size_t want = previousRowCount + std::max<std::size_t>(200, previousRowCount / 10);
+    return static_cast<int>(std::clamp<std::size_t>(want, kTransparentPageMin, kTransparentPageMax));
+}
+
+bool sentSweepIsFull(int tip, int fromHeight)
+{
+    return tip <= 0 || fromHeight < 0 || fromHeight > tip;
+}
+
+nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight)
+{
+    nlohmann::json params = nlohmann::json::array();
+    params.push_back(anyZAddress);
+    params.push_back(0);
+    if (sentSweepIsFull(tip, fromHeight)) return params;
+    const int maxDepth = (tip - fromHeight) + kSentSweepReorgMarginBlocks;
+    params.push_back(2);          // filter type 2: depth
+    params.push_back(maxDepth);   // at most this deep (mempool txs, depth 0, always pass)
+    return params;
+}
+
+long fullSentSweepRetryDelaySec(int consecutiveFailures)
+{
+    if (consecutiveFailures <= 0) return 0;
+    if (consecutiveFailures == 1) return 600;
+    if (consecutiveFailures == 2) return 3600;
+    return 21600;
+}
+
+bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind)
+{
+    using K = rpc::RpcFailureKind;
+    return kind == K::ReplyTimeout || kind == K::AppError || kind == K::Other;
+}
+
+std::size_t oldestScannedAddressIndex(const std::vector<std::string>& addresses,
+                                      const std::unordered_map<std::string, int>& heights)
+{
+    std::size_t best = 0;
+    int bestHeight = 0;
+    bool haveBest = false;
+    for (std::size_t i = 0; i < addresses.size(); ++i) {
+        const auto it = heights.find(addresses[i]);
+        if (it == heights.end()) return i;   // never scanned: first
+        if (!haveBest || it->second < bestHeight) {
+            best = i;
+            bestHeight = it->second;
+            haveBest = true;
+        }
+    }
+    return best;
+}
+
+int ShieldedHistoryPass::floorAt(int tip) const
+{
+    return tip > 0 ? std::min(startHeight_, tip) : startHeight_;
+}
+
+ShieldedHistoryPass::Work ShieldedHistoryPass::evaluate(int tip, const std::vector<std::string>& addresses,
+                                                        const std::unordered_map<std::string, int>& heights,
+                                                        int staleAfter, bool initialComplete) const
+{
+    Work work;
+    work.total = addresses.size();
+    const int floor = active_ ? floorAt(tip) : -1;
+    std::size_t below = 0;
+    bool staleDue = false;
+    for (const auto& address : addresses) {
+        const auto it = heights.find(address);
+        if (it == heights.end()) {
+            ++work.missing;
+            continue;
+        }
+        if (active_) {
+            if (it->second < floor) ++below;
+        } else if (tip > 0 && it->second < tip - staleAfter) {
+            staleDue = true;
+        }
+    }
+    work.pending = work.missing + (active_ ? below : (staleDue ? work.total - work.missing : 0));
+    work.explicitWork = !stalled() &&
+        (work.missing > 0 || (active_ ? (explicit_ && below > 0) : (staleDue && !initialComplete)));
+    work.routineWork = !work.explicitWork && work.pending > 0;
+    return work;
+}
+
+void ShieldedHistoryPass::begin(int tip, const std::vector<std::string>& addresses,
+                                const std::unordered_map<std::string, int>& heights, int staleAfter,
+                                bool initialComplete, bool routineAllowed)
+{
+    if (active_) {
+        if (tip > 0 && startHeight_ > tip) startHeight_ = tip;   // the chain reorganised below the floor
+        return;
+    }
+    if (tip <= 0 || !(routineAllowed || !initialComplete)) return;
+    bool staleDue = false;
+    for (const auto& address : addresses) {
+        const auto it = heights.find(address);
+        if (it != heights.end() && it->second < tip - staleAfter) {
+            staleDue = true;
+            break;
+        }
+    }
+    if (!staleDue) return;
+    active_ = true;
+    explicit_ = !initialComplete;
+    startHeight_ = tip;
+}
+
+int ShieldedHistoryPass::scanFloor(int tip, bool routineAllowed) const
+{
+    return (active_ && (explicit_ || routineAllowed)) ? floorAt(tip) : -1;
+}
+
+void ShieldedHistoryPass::finish(int tip, const std::vector<std::string>& addresses,
+                                 const std::unordered_map<std::string, int>& heights, std::size_t attempted,
+                                 std::size_t succeeded)
+{
+    if (attempted > 0) stalledCycles_ = succeeded > 0 ? 0 : stalledCycles_ + 1;
+    // An empty list proves nothing (a failed enumeration blanks it); a real wallet change resets the pass.
+    if (!active_ || addresses.empty()) return;
+    const int floor = floorAt(tip);
+    for (const auto& address : addresses) {
+        const auto it = heights.find(address);
+        if (it == heights.end() || it->second < floor) return;   // still work left in this pass
+    }
+    active_ = false;
+    explicit_ = false;
+}
+
+void ShieldedHistoryPass::reset()
+{
+    *this = ShieldedHistoryPass{};
+}
+
+TxCycle decideTxCycle(const TxCycleInputs& in)
+{
+    const bool wanted = in.neverFetched || in.newTip || in.dirty || in.explicitScanWork || in.routineScanWork;
+    const bool scansAllowed = !in.syncing && in.heightKnown;
+    if (wanted) {
+        if (scansAllowed && (in.explicitScanWork || in.gateOpen)) return TxCycle::Full;
+        // Rows now (a mutation, a tracked send at a new tip); wallet-wide scans wait for the throttle.
+        if (in.neverFetched || in.dirty || (in.sendTracking && in.newTip)) return TxCycle::Light;
+    }
+    if (in.recentDue) return (scansAllowed && in.gateOpen) ? TxCycle::RecentWithProbe : TxCycle::Recent;
+    return TxCycle::None;
+}
+
+}  // namespace services
+}  // namespace dragonx