Files
ObsidianDragon/src/services/history_scan_policy.cpp
T
DanSandClaude Opus 4.8 9ff64b1846 perf(history): band the full sent-sweep so it doesn't hold cs_main for minutes
A full z_listsentbyaddress holds the daemon's cs_main for the whole sweep --
~140 s on a large wallet on a daemon without the sent-history decrypt cache --
which starves block connection and makes every other RPC (getinfo, balances)
wait, so the node looks "busy". This runs the full sweep in depth bands so no
single call holds the lock for long.

- The full case now covers [0, tip] in bands: z_listsentbyaddress(z, minconf=a,
  filter-2, maxDepth=b) returns exactly depth [a, b]. Newest band first (its
  minconf=0 includes the mempool); results dedupe by txid across the overlap.
- Each band is a separate call, so cs_main is held only ~5 s at a time and the
  node stays responsive between bands (block connection isn't starved, the busy
  canary answers). Total CPU is unchanged -- just not in one long lock.
- sentSweepNextBandSize adapts the next band toward ~5 s: grows up to 4x in
  sparse depth (empty regions covered fast), shrinks to 1/2x in dense depth.
- W advances only after every band succeeds; a mid-sweep failure re-sweeps
  rather than marking a partial result. The windowed sweep is unchanged.
- Each band also fits the 120 s timeout on its own, so the seed completes even
  on a daemon where the single whole-wallet call used to time out.

This removes the ~2.5 min hang on the first-run seed, the weekly safety-net
re-sweep, and any invalidation. Multi-band + band-sizing covered by tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-10-17 14:42:16 -05:00

243 lines
8.7 KiB
C++

// DragonX Wallet - ImGui Edition
// Copyright 2024-2026 The Hush Developers
// Released under the GPLv3
#include "history_scan_policy.h"
#include <algorithm>
#include <cmath>
namespace dragonx {
namespace services {
int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs)
{
if (addressCount == 0 || !(perAddressScanMs > 0.0)) return kHistoryMinStaleBlocks;
const double blocks = std::ceil((static_cast<double>(addressCount) * perAddressScanMs / 1000.0) /
(kHistoryBlockSeconds * kHistoryScanDuty));
if (!(blocks < kHistoryMaxStaleBlocks)) return kHistoryMaxStaleBlocks; // also NaN / inf
return std::max(kHistoryMinStaleBlocks, static_cast<int>(blocks));
}
double updateScanCostEstimateMs(double previousMs, double sampleMs)
{
if (!(sampleMs > 0.0)) return previousMs;
if (!(previousMs > 0.0) || sampleMs > previousMs) return sampleMs;
return 0.75 * previousMs + 0.25 * sampleMs;
}
int transparentPageSize(std::size_t previousRowCount)
{
const std::size_t want = previousRowCount + std::max<std::size_t>(200, previousRowCount / 10);
return static_cast<int>(std::clamp<std::size_t>(want, kTransparentPageMin, kTransparentPageMax));
}
bool sentSweepIsFull(int tip, int fromHeight)
{
return tip <= 0 || fromHeight < 0 || fromHeight > tip;
}
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight, int margin)
{
nlohmann::json params = nlohmann::json::array();
params.push_back(anyZAddress);
params.push_back(0);
if (sentSweepIsFull(tip, fromHeight)) return params;
const int maxDepth = (tip - fromHeight) + std::max(0, margin);
params.push_back(2); // filter type 2: depth
params.push_back(maxDepth); // at most this deep (mempool txs, depth 0, always pass)
return params;
}
int sentSweepMargin(bool firstAfterLoad)
{
return firstAfterLoad ? (kSentSweepReorgMarginBlocks + kSentSweepFirstLoadExtraBlocks)
: kSentSweepReorgMarginBlocks;
}
int sentSweepAnchor(int fromHeight, bool firstAfterLoad)
{
return std::max(1, fromHeight - sentSweepMargin(firstAfterLoad));
}
bool sentSweepNeedsLongCall(int tip, int fromHeight)
{
return tip > 0 && fromHeight >= 0 && (tip - fromHeight) > kSentSweepLongCallBlocks;
}
bool sentSweepSafetyNetDue(std::int64_t nowSec, std::int64_t lastFullSec)
{
return lastFullSec > 0 && (nowSec - lastFullSec) > kSentSweepSafetyNetSec;
}
int sentSweepNextBandSize(int prevSize, double prevMs)
{
if (!(prevMs > 0.0)) return std::clamp(prevSize, kSentSweepBandMinBlocks, kSentSweepBandMaxBlocks);
double scale = static_cast<double>(kSentSweepBandTargetMs) / prevMs;
scale = std::clamp(scale, 0.5, 4.0); // don't swing more than 2x smaller / 4x larger per step
int next = static_cast<int>(std::lround(prevSize * scale));
return std::clamp(next, kSentSweepBandMinBlocks, kSentSweepBandMaxBlocks);
}
long fullSentSweepRetryDelaySec(int consecutiveFailures)
{
if (consecutiveFailures <= 0) return 0;
if (consecutiveFailures == 1) return 600;
if (consecutiveFailures == 2) return 3600;
return 21600;
}
bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind)
{
using K = rpc::RpcFailureKind;
return kind == K::ReplyTimeout || kind == K::AppError || kind == K::Other;
}
std::size_t oldestScannedAddressIndex(const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights)
{
std::size_t best = 0;
int bestHeight = 0;
bool haveBest = false;
for (std::size_t i = 0; i < addresses.size(); ++i) {
const auto it = heights.find(addresses[i]);
if (it == heights.end()) return i; // never scanned: first
if (!haveBest || it->second < bestHeight) {
best = i;
bestHeight = it->second;
haveBest = true;
}
}
return best;
}
std::string ProbeRotation::next(const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights)
{
std::unordered_set<std::string> listed(addresses.begin(), addresses.end());
for (auto it = left_.begin(); it != left_.end();) {
if (listed.count(*it)) ++it;
else it = left_.erase(it);
}
if (left_.empty()) left_ = std::move(listed);
std::vector<std::string> left;
for (const auto& address : addresses) {
if (left_.count(address)) left.push_back(address);
}
if (left.empty()) return {};
return left[oldestScannedAddressIndex(left, heights)];
}
int ShieldedHistoryPass::floorAt(int tip) const
{
return tip > 0 ? std::min(startHeight_, tip) : startHeight_;
}
ShieldedHistoryPass::Work ShieldedHistoryPass::evaluate(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights,
int staleAfter, bool initialComplete) const
{
Work work;
work.total = addresses.size();
const int floor = active_ ? floorAt(tip) : -1;
std::size_t below = 0;
bool staleDue = false;
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it == heights.end()) {
++work.missing;
continue;
}
if (active_) {
if (it->second < floor) ++below;
} else if (tip > 0 && it->second < tip - staleAfter) {
staleDue = true;
}
}
work.pending = work.missing + (active_ ? below : (staleDue ? work.total - work.missing : 0));
work.explicitWork = !stalled() &&
(work.missing > 0 || (active_ ? (explicit_ && below > 0) : (staleDue && !initialComplete)));
work.routineWork = !work.explicitWork && work.pending > 0;
return work;
}
void ShieldedHistoryPass::begin(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights, int staleAfter,
bool initialComplete, bool routineAllowed)
{
if (active_) {
if (tip > 0 && startHeight_ > tip) startHeight_ = tip; // the chain reorganised below the floor
return;
}
if (tip <= 0 || !(routineAllowed || !initialComplete)) return;
bool staleDue = false;
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it != heights.end() && it->second < tip - staleAfter) {
staleDue = true;
break;
}
}
if (!staleDue) return;
active_ = true;
explicit_ = !initialComplete;
startHeight_ = tip;
}
int ShieldedHistoryPass::scanFloor(int tip, bool routineAllowed) const
{
return (active_ && (explicit_ || routineAllowed)) ? floorAt(tip) : -1;
}
void ShieldedHistoryPass::finish(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights, std::size_t attempted,
std::size_t succeeded)
{
if (attempted > 0) stalledCycles_ = succeeded > 0 ? 0 : stalledCycles_ + 1;
// An empty list proves nothing (a failed enumeration blanks it); a real wallet change resets the pass.
if (!active_ || addresses.empty()) return;
const int floor = floorAt(tip);
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it == heights.end() || it->second < floor) return; // still work left in this pass
}
active_ = false;
explicit_ = false;
}
void ShieldedHistoryPass::reset()
{
*this = ShieldedHistoryPass{};
}
TxCycle decideTxCycle(const TxCycleInputs& in)
{
const bool wanted = in.neverFetched || in.newTip || in.dirty || in.explicitScanWork || in.routineScanWork;
const bool scansAllowed = !in.syncing && in.heightKnown;
if (wanted) {
if (scansAllowed && (in.explicitScanWork || in.gateOpen)) return TxCycle::Full;
// Rows now (a mutation, a tracked send at a new tip); wallet-wide scans wait for the throttle.
if (in.neverFetched || in.dirty || (in.sendTracking && in.newTip)) return TxCycle::Light;
}
if (in.recentDue) return (scansAllowed && in.gateOpen) ? TxCycle::RecentWithProbe : TxCycle::Recent;
return TxCycle::None;
}
TxCycleInputs shapeTxCycleForMiningPage(TxCycleInputs in)
{
if (!in.sendTracking) { // tracked sends need their Light cycle at a new tip, wherever the user is
in.newTip = false;
in.routineScanWork = false;
}
return in;
}
bool miningTxPollDue(const MiningTxPollInputs& in)
{
return in.historyLoaded && in.tip > 0 && in.tip != in.lastPolledTip && in.txAgeDue && !in.dirty &&
!in.addressesDirty;
}
} // namespace services
} // namespace dragonx