perf(history): persist the sent-sweep watermark (windowed relaunch)
z_listsentbyaddress re-walks the whole wallet when there's no usable watermark, which the daemon does under cs_main for every send -- hundreds of seconds, 400+ s and ~19 GB RSS on a large wallet. The watermark lived only in memory, so every relaunch, reconnect or daemon restart paid for a full sweep. This persists it so those cases run a bounded windowed sweep ([z, 0, 2, tip - W + margin]) instead (SENT-HISTORY-SWEEP Part 1). - The watermark (W = last swept tip), its anchor (A = W - margin) and anchor hash (H_A = getblockhash(A)), plus the enrichment backlog and the full- sweep backoff, ride in the encrypted history snapshot as an optional `sent_sweep` object (schema stays v1; a missing/inconsistent one reads as "no watermark" = today's behaviour, so older builds and payloads are safe). - Before the first windowed sweep after a load, the anchor is verified (getblockhash(A) == H_A). A match means any fork is inside the window; a mismatch (reorg deeper than the margin, different chain/datadir) or an inability to check drops the watermark and sweeps the whole wallet. - The first sweep after a load uses a bigger margin (absorbs notifier lag), and a watermark older than 2400 blocks uses the long timeout (callLong). - The backlog is persisted WITH W: a pure z->z send's txid lives only in a sweep result and would be lost if the window moved past an un-enriched one. - Watermark changes reach disk (folded into the store signature), and every hard case still forces a full sweep: reorg (anchor verify), GUI rescan / zap / import / address change, restoreOriginalWallet, and disconnect. Deferred (follow-ups, each only turns a correct-but-slow case faster, never a correctness risk): the banded full sweep for seeding on a pre-note-cache daemon, the external-import tripwire, and the weekly safety-net re-sweep. Part 2 (the daemon's decrypt cost) is already on the daemon's perf/sapling-note-cache branch. Pure helpers + cache round-trip + anchor-verify/large-window covered by tests; the getblockhash-after-sweep updates the pinned RPC sequences. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1761,7 +1761,10 @@ private:
|
||||
services::ShieldedHistoryPass history_pass_;
|
||||
std::uint64_t history_scan_epoch_ = 0; // bumped when the scan heights are invalidated
|
||||
std::uint64_t history_sent_epoch_ = 0; // bumped when the sent sweep's watermark is reset
|
||||
int history_sent_scan_height_ = -1; // tip of the last successful z_listsentbyaddress; -1 = none
|
||||
int history_sent_scan_height_ = -1; // W: tip of the last successful z_listsentbyaddress; -1 = none
|
||||
int history_sent_anchor_height_ = -1; // A: bottom edge the last sweep's window covered (W - margin)
|
||||
std::string history_sent_anchor_hash_; // H_A: getblockhash(A) at that sweep; re-verified after a load
|
||||
bool history_sent_verify_pending_ = false; // a loaded watermark owes an anchor verify before its next sweep
|
||||
int history_full_sweep_failures_ = 0; // whole-wallet sweeps failed in a row (backoff)
|
||||
std::time_t history_full_sweep_retry_at_ = 0;
|
||||
std::unordered_set<std::string> history_enrich_backlog_; // txids found, not enriched yet
|
||||
|
||||
+46
-2
@@ -1888,6 +1888,9 @@ void App::invalidateShieldedHistoryScanProgress(bool persistCache, bool keepSent
|
||||
recent_probe_rotation_.reset();
|
||||
if (!keepSentScan) {
|
||||
history_sent_scan_height_ = -1; // the next sweep covers the whole wallet again
|
||||
history_sent_anchor_height_ = -1;
|
||||
history_sent_anchor_hash_.clear();
|
||||
history_sent_verify_pending_ = false;
|
||||
history_enrich_backlog_.clear();
|
||||
history_full_sweep_failures_ = 0;
|
||||
history_full_sweep_retry_at_ = 0;
|
||||
@@ -2007,11 +2010,26 @@ void App::loadTransactionHistoryCacheIfAvailable()
|
||||
}
|
||||
}
|
||||
confirmed_cache_block_ = loaded.tipHeight;
|
||||
// Restore the sent-sweep watermark so the first sweep is windowed, not a 400 s full sweep. A verify is
|
||||
// owed before the window is trusted (the next sweep checks getblockhash(A) == H_A), and the enrichment
|
||||
// backlog must come back too: a pure z->z send's txid lives only in a sweep result, and would be lost if
|
||||
// the window moved past an un-enriched one.
|
||||
history_sent_scan_height_ = loaded.sentSweep.height;
|
||||
history_sent_anchor_height_ = loaded.sentSweep.anchorHeight;
|
||||
history_sent_anchor_hash_ = loaded.sentSweep.anchorHash;
|
||||
history_sent_verify_pending_ = loaded.sentSweep.valid();
|
||||
history_enrich_backlog_.clear();
|
||||
history_enrich_backlog_.insert(loaded.sentSweep.backlog.begin(), loaded.sentSweep.backlog.end());
|
||||
history_full_sweep_failures_ = loaded.sentSweep.fullFailures;
|
||||
history_full_sweep_retry_at_ = loaded.sentSweep.fullRetryDelaySec > 0
|
||||
? std::time(nullptr) + loaded.sentSweep.fullRetryDelaySec : 0;
|
||||
++history_sent_epoch_; // the loaded watermark replaces any in-flight sweep's result
|
||||
transaction_history_cache_loaded_ = true;
|
||||
// What is on disk is exactly what was just loaded: the first identical refresh must not rewrite it.
|
||||
history_store_.sig = data::historySnapshotSignature(
|
||||
state_.transactions, {}, data::TransactionHistoryCache::walletIdentityHash(walletIdentity),
|
||||
transaction_history_cache_.keyId(), shielded_history_scan_heights_, loaded.tipHeight, loaded.tipHash);
|
||||
transaction_history_cache_.keyId(), shielded_history_scan_heights_, loaded.tipHeight, loaded.tipHash,
|
||||
history_sent_scan_height_, history_sent_anchor_height_, history_sent_anchor_hash_, history_enrich_backlog_);
|
||||
history_store_.have = true;
|
||||
history_store_.lastSubmit = std::chrono::steady_clock::now();
|
||||
history_store_pending_ = false;
|
||||
@@ -2040,7 +2058,9 @@ void App::storeTransactionHistoryCacheIfAvailable(data::HistoryStoreMode mode)
|
||||
state_.transactions, pendingOpids,
|
||||
data::TransactionHistoryCache::walletIdentityHash(walletIdentity),
|
||||
transaction_history_cache_.keyId(), shielded_history_scan_heights_,
|
||||
state_.sync.blocks, state_.sync.best_blockhash);
|
||||
state_.sync.blocks, state_.sync.best_blockhash,
|
||||
history_sent_scan_height_, history_sent_anchor_height_, history_sent_anchor_hash_,
|
||||
history_enrich_backlog_);
|
||||
if (sig.rows == 0) return;
|
||||
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
@@ -2077,6 +2097,18 @@ void App::storeTransactionHistoryCacheIfAvailable(data::HistoryStoreMode mode)
|
||||
if (pendingOpids.find(transaction.txid) != pendingOpids.end()) continue;
|
||||
job->transactions.push_back(transaction);
|
||||
}
|
||||
// The sent-sweep watermark + enrichment backlog travel with the rows (writeSnapshot emits them only
|
||||
// when self-consistent). The backoff is stored as a delay relative to now (retry_at is absolute).
|
||||
job->sentSweep.height = history_sent_scan_height_;
|
||||
job->sentSweep.anchorHeight = history_sent_anchor_height_;
|
||||
job->sentSweep.anchorHash = history_sent_anchor_hash_;
|
||||
job->sentSweep.backlog.assign(history_enrich_backlog_.begin(), history_enrich_backlog_.end());
|
||||
job->sentSweep.fullFailures = history_full_sweep_failures_;
|
||||
{
|
||||
const std::time_t nowSec = std::time(nullptr);
|
||||
job->sentSweep.fullRetryDelaySec = history_full_sweep_retry_at_ > nowSec
|
||||
? static_cast<long>(history_full_sweep_retry_at_ - nowSec) : 0;
|
||||
}
|
||||
|
||||
auto* cache = &transaction_history_cache_; // App member: outlives every job (see app.h)
|
||||
history_writer_.submit("tx-history", [cache, job]() { return cache->writeSnapshot(std::move(*job)); });
|
||||
@@ -2592,6 +2624,9 @@ void App::refreshAddressData(services::ScanTicket ticket)
|
||||
}
|
||||
if (newUnscannedZ && (!previousZ.empty() || !history_listed_z_.empty())) {
|
||||
history_sent_scan_height_ = -1;
|
||||
history_sent_anchor_height_ = -1;
|
||||
history_sent_anchor_hash_.clear();
|
||||
history_sent_verify_pending_ = false;
|
||||
history_full_sweep_failures_ = 0;
|
||||
history_full_sweep_retry_at_ = 0;
|
||||
++history_sent_epoch_;
|
||||
@@ -2777,6 +2812,9 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi
|
||||
}
|
||||
}
|
||||
transactionSnapshot.sentScanFromHeight = history_sent_scan_height_;
|
||||
transactionSnapshot.sentVerifyPending = history_sent_verify_pending_;
|
||||
transactionSnapshot.sentVerifyAnchorHeight = history_sent_anchor_height_;
|
||||
transactionSnapshot.sentVerifyAnchorHash = history_sent_anchor_hash_;
|
||||
// A windowed sweep mid-pass too, now and then, so an untracked send (the Console, another wallet on
|
||||
// the same seed) doesn't wait for a whole pass on a large wallet: at most every quarter of the pass
|
||||
// cadence, and only in cycles the throttle allows.
|
||||
@@ -2844,6 +2882,9 @@ void App::startTransactionCycle(bool full, bool routineAllowed, services::ScanTi
|
||||
if (sameSentEpoch) {
|
||||
if (result.sentSweepOk && result.sentSweepHeight > 0) {
|
||||
history_sent_scan_height_ = result.sentSweepHeight;
|
||||
history_sent_anchor_height_ = result.sentSweepAnchorHeight;
|
||||
history_sent_anchor_hash_ = result.sentSweepAnchorHash;
|
||||
history_sent_verify_pending_ = false; // this sweep covered the window; nothing owed
|
||||
history_full_sweep_failures_ = 0;
|
||||
history_full_sweep_retry_at_ = 0;
|
||||
} else if (result.sentSweepRan && result.sentSweepFull &&
|
||||
@@ -6562,6 +6603,9 @@ void App::restoreOriginalWallet()
|
||||
connection_status_ = TR("sb_restarting_daemon");
|
||||
abortMaintenanceCalls(); // a long call holds the curl lock disconnect() needs
|
||||
if (rpc_ && rpc_->isConnected()) rpc_->disconnect();
|
||||
// Swapping the wallet + restarting with -rescan changes the history: invalidate the sent-sweep watermark
|
||||
// (and persist that) before the disconnect, as rescanBlockchain() and the zap path already do.
|
||||
invalidateShieldedHistoryScanProgress(true);
|
||||
onDisconnected("Restoring original wallet");
|
||||
ui::Notifications::instance().info(TR("wallet_restore_started"), 12.0f);
|
||||
|
||||
|
||||
@@ -366,6 +366,27 @@ TransactionHistoryCache::LoadResult TransactionHistoryCache::load(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sent-sweep watermark (optional). Drop it unless W/A are self-consistent and W is not above the
|
||||
// stored tip (a replay after an unclean stop would otherwise use a stale watermark); the anchor
|
||||
// hash is re-verified against the live chain on the next sweep regardless.
|
||||
if (payload.contains("sent_sweep") && payload["sent_sweep"].is_object()) {
|
||||
const auto& s = payload["sent_sweep"];
|
||||
SentSweepState sw;
|
||||
sw.height = s.value("height", -1);
|
||||
sw.anchorHeight = s.value("anchor_height", -1);
|
||||
sw.anchorHash = s.value("anchor_hash", std::string());
|
||||
sw.fullFailures = s.value("full_failures", 0);
|
||||
sw.fullRetryDelaySec = static_cast<long>(s.value("full_retry_delay_s", std::int64_t{0}));
|
||||
sw.daemonId = s.value("daemon_id", std::string());
|
||||
if (s.contains("backlog") && s["backlog"].is_array()) {
|
||||
for (const auto& txid : s["backlog"])
|
||||
if (txid.is_string() && !txid.get<std::string>().empty())
|
||||
sw.backlog.push_back(txid.get<std::string>());
|
||||
}
|
||||
const bool consistent = sw.height >= 0 && sw.anchorHeight > 0 &&
|
||||
sw.anchorHeight <= sw.height && sw.height <= tipHeight;
|
||||
if (consistent) result.sentSweep = std::move(sw);
|
||||
}
|
||||
result.tipHeight = tipHeight;
|
||||
result.tipHash = tipHash;
|
||||
result.updatedAt = updatedAt;
|
||||
@@ -383,7 +404,8 @@ bool TransactionHistoryCache::replace(const std::string& walletIdentity,
|
||||
const std::string& tipHash,
|
||||
const std::vector<TransactionInfo>& transactions,
|
||||
std::time_t updatedAt,
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights)
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights,
|
||||
const SentSweepState& sentSweep)
|
||||
{
|
||||
// The synchronous form (tests, tools): the same write the background writer does.
|
||||
SnapshotJob job;
|
||||
@@ -394,6 +416,7 @@ bool TransactionHistoryCache::replace(const std::string& walletIdentity,
|
||||
job.transactions = transactions;
|
||||
job.updatedAt = updatedAt;
|
||||
job.shieldedScanHeights = shieldedScanHeights;
|
||||
job.sentSweep = sentSweep;
|
||||
return writeSnapshot(std::move(job));
|
||||
}
|
||||
|
||||
@@ -426,6 +449,23 @@ bool TransactionHistoryCache::writeSnapshot(SnapshotJob&& job)
|
||||
}
|
||||
}
|
||||
|
||||
// Sent-sweep watermark, only when it is self-consistent (0 < A <= W <= tip). An unknown key means
|
||||
// W = -1 on load, which is the pre-watermark behaviour, so older builds read a new payload safely.
|
||||
const SentSweepState& sw = job.sentSweep;
|
||||
if (sw.height >= 0 && sw.anchorHeight > 0 && sw.anchorHeight <= sw.height && sw.height <= job.tipHeight) {
|
||||
json s;
|
||||
s["height"] = sw.height;
|
||||
s["anchor_height"] = sw.anchorHeight;
|
||||
s["anchor_hash"] = sw.anchorHash;
|
||||
s["backlog"] = json::array();
|
||||
for (const auto& txid : sw.backlog)
|
||||
if (!txid.empty()) s["backlog"].push_back(txid);
|
||||
s["full_failures"] = sw.fullFailures;
|
||||
s["full_retry_delay_s"] = static_cast<std::int64_t>(sw.fullRetryDelaySec);
|
||||
s["daemon_id"] = sw.daemonId;
|
||||
payload["sent_sweep"] = std::move(s);
|
||||
}
|
||||
|
||||
std::string plainText = payload.dump();
|
||||
std::vector<unsigned char> nonce;
|
||||
std::vector<unsigned char> cipherText;
|
||||
|
||||
@@ -15,6 +15,24 @@ struct sqlite3;
|
||||
namespace dragonx {
|
||||
namespace data {
|
||||
|
||||
// Persisted z_listsentbyaddress watermark, so a relaunch runs a windowed sweep instead of a full one.
|
||||
// height (W) = tip of the last successful sweep; anchorHeight (A) = W - margin, the bottom edge the window
|
||||
// covered; anchorHash (H_A) = getblockhash(A) at that time, verified on the next load. backlog = txids found
|
||||
// but not yet enriched (must persist WITH W, or a pure z->z send whose txid lives only in a sweep result is
|
||||
// lost once the window moves past it). The backoff fields keep a relaunch from immediately re-running a full
|
||||
// sweep that keeps failing; daemonId resets the backoff on a version change. (Namespace-scope, not nested, so
|
||||
// it is a complete type for the default argument of replace().)
|
||||
struct SentSweepState {
|
||||
int height = -1;
|
||||
int anchorHeight = -1;
|
||||
std::string anchorHash;
|
||||
std::vector<std::string> backlog;
|
||||
int fullFailures = 0;
|
||||
long fullRetryDelaySec = 0;
|
||||
std::string daemonId;
|
||||
bool valid() const { return height >= 0; }
|
||||
};
|
||||
|
||||
class TransactionHistoryCache {
|
||||
public:
|
||||
struct LoadResult {
|
||||
@@ -25,6 +43,7 @@ public:
|
||||
std::time_t updatedAt = 0;
|
||||
std::vector<TransactionInfo> transactions;
|
||||
std::unordered_map<std::string, int> shieldedScanHeights;
|
||||
SentSweepState sentSweep;
|
||||
};
|
||||
|
||||
// Everything one snapshot write needs, BY VALUE, so it can run on the history writer thread after the UI
|
||||
@@ -38,6 +57,7 @@ public:
|
||||
std::vector<TransactionInfo> transactions;
|
||||
std::time_t updatedAt = 0;
|
||||
std::unordered_map<std::string, int> shieldedScanHeights;
|
||||
SentSweepState sentSweep;
|
||||
|
||||
SnapshotJob() = default;
|
||||
SnapshotJob(const SnapshotJob&) = delete;
|
||||
@@ -81,7 +101,8 @@ public:
|
||||
const std::string& tipHash,
|
||||
const std::vector<TransactionInfo>& transactions,
|
||||
std::time_t updatedAt,
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights = {});
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights = {},
|
||||
const SentSweepState& sentSweep = SentSweepState{});
|
||||
// Thread-safe snapshot write (the history writer thread; replace() wraps it): JSON + AEAD with no lock
|
||||
// held, then INSERT OR REPLACE + prune under the DB lock. Always wipes job.key. False for a keyless job.
|
||||
bool writeSnapshot(SnapshotJob&& job);
|
||||
|
||||
@@ -65,7 +65,11 @@ HistorySnapshotSig historySnapshotSignature(const std::vector<TransactionInfo>&
|
||||
std::uint64_t keyId,
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights,
|
||||
int tipHeight,
|
||||
const std::string& tipHash)
|
||||
const std::string& tipHash,
|
||||
int sentHeight,
|
||||
int sentAnchorHeight,
|
||||
const std::string& sentAnchorHash,
|
||||
const std::unordered_set<std::string>& sentBacklog)
|
||||
{
|
||||
HistorySnapshotSig sig;
|
||||
std::uint64_t rowSum = 0; // sums: commutative, so row order does not matter; duplicates still count
|
||||
@@ -89,11 +93,19 @@ HistorySnapshotSig historySnapshotSignature(const std::vector<TransactionInfo>&
|
||||
exactSum += finalize(e);
|
||||
++sig.rows;
|
||||
}
|
||||
std::uint64_t backlogSum = 0; // order-independent: a set, so sum is commutative
|
||||
for (const auto& txid : sentBacklog) {
|
||||
if (txid.empty()) continue;
|
||||
std::uint64_t h = kOffset;
|
||||
mixStr(h, txid);
|
||||
backlogSum += finalize(h);
|
||||
}
|
||||
std::uint64_t c = kOffset;
|
||||
mixU64(c, rowSum);
|
||||
mixU64(c, sig.rows);
|
||||
mixStr(c, walletHash);
|
||||
mixU64(c, keyId);
|
||||
mixU64(c, backlogSum);
|
||||
sig.content = finalize(c);
|
||||
|
||||
std::uint64_t heightSum = 0;
|
||||
@@ -109,6 +121,9 @@ HistorySnapshotSig historySnapshotSignature(const std::vector<TransactionInfo>&
|
||||
mixU64(s, exactSum);
|
||||
mixU64(s, static_cast<std::uint64_t>(static_cast<std::int64_t>(tipHeight)));
|
||||
mixStr(s, tipHash);
|
||||
mixU64(s, static_cast<std::uint64_t>(static_cast<std::int64_t>(sentHeight)));
|
||||
mixU64(s, static_cast<std::uint64_t>(static_cast<std::int64_t>(sentAnchorHeight)));
|
||||
mixStr(s, sentAnchorHash);
|
||||
sig.slow = finalize(s);
|
||||
return sig;
|
||||
}
|
||||
|
||||
@@ -38,7 +38,15 @@ HistorySnapshotSig historySnapshotSignature(const std::vector<TransactionInfo>&
|
||||
std::uint64_t keyId,
|
||||
const std::unordered_map<std::string, int>& shieldedScanHeights,
|
||||
int tipHeight,
|
||||
const std::string& tipHash);
|
||||
const std::string& tipHash,
|
||||
// Sent-sweep watermark: W/A/H_A move per sweep (folded into `slow`,
|
||||
// written on the slow cadence and always on disconnect/lock/exit),
|
||||
// the enrichment backlog into `content` (written within ~20 s), so an
|
||||
// invalidation that only changes the watermark still reaches disk.
|
||||
int sentHeight = -1,
|
||||
int sentAnchorHeight = -1,
|
||||
const std::string& sentAnchorHash = std::string(),
|
||||
const std::unordered_set<std::string>& sentBacklog = {});
|
||||
|
||||
enum class HistoryStoreMode {
|
||||
Coalesce, // routine refresh: content changes at most every kHistoryMinStoreIntervalSec, slow drift every
|
||||
|
||||
@@ -37,18 +37,34 @@ bool sentSweepIsFull(int tip, int fromHeight)
|
||||
return tip <= 0 || fromHeight < 0 || fromHeight > tip;
|
||||
}
|
||||
|
||||
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight)
|
||||
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight, int margin)
|
||||
{
|
||||
nlohmann::json params = nlohmann::json::array();
|
||||
params.push_back(anyZAddress);
|
||||
params.push_back(0);
|
||||
if (sentSweepIsFull(tip, fromHeight)) return params;
|
||||
const int maxDepth = (tip - fromHeight) + kSentSweepReorgMarginBlocks;
|
||||
const int maxDepth = (tip - fromHeight) + std::max(0, margin);
|
||||
params.push_back(2); // filter type 2: depth
|
||||
params.push_back(maxDepth); // at most this deep (mempool txs, depth 0, always pass)
|
||||
return params;
|
||||
}
|
||||
|
||||
int sentSweepMargin(bool firstAfterLoad)
|
||||
{
|
||||
return firstAfterLoad ? (kSentSweepReorgMarginBlocks + kSentSweepFirstLoadExtraBlocks)
|
||||
: kSentSweepReorgMarginBlocks;
|
||||
}
|
||||
|
||||
int sentSweepAnchor(int fromHeight, bool firstAfterLoad)
|
||||
{
|
||||
return std::max(1, fromHeight - sentSweepMargin(firstAfterLoad));
|
||||
}
|
||||
|
||||
bool sentSweepNeedsLongCall(int tip, int fromHeight)
|
||||
{
|
||||
return tip > 0 && fromHeight >= 0 && (tip - fromHeight) > kSentSweepLongCallBlocks;
|
||||
}
|
||||
|
||||
long fullSentSweepRetryDelaySec(int consecutiveFailures)
|
||||
{
|
||||
if (consecutiveFailures <= 0) return 0;
|
||||
|
||||
@@ -28,6 +28,9 @@ constexpr double kHistoryScanDuty = 0.10; // the same 10% as the othe
|
||||
constexpr int kHistoryMinStaleBlocks = 8; // re-scan an address at most every 8 blocks...
|
||||
constexpr int kHistoryMaxStaleBlocks = 500; // ...and at least every 500 (a safety net)
|
||||
constexpr int kSentSweepReorgMarginBlocks = 100; // > MAX_REORG_LENGTH (99), plus tip drift
|
||||
constexpr int kSentSweepFirstLoadExtraBlocks = 1000; // E: extra margin on the first sweep after a load,
|
||||
// to absorb wallet-notifier lag at the last save
|
||||
constexpr int kSentSweepLongCallBlocks = 2400; // tip - W beyond this: use callLong + count failures
|
||||
constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish
|
||||
constexpr int kTransparentPageMin = 1000;
|
||||
constexpr int kTransparentPageMax = 5000;
|
||||
@@ -50,8 +53,17 @@ int transparentPageSize(std::size_t previousRowCount);
|
||||
// tip after a reorg), else only txs at most (tip - fromHeight) + kSentSweepReorgMarginBlocks deep.
|
||||
bool sentSweepIsFull(int tip, int fromHeight);
|
||||
// [z, 0] for the full sweep, [z, 0, 2, maxDepth] for the window: JSON integers, and never 3 params (the
|
||||
// daemon rejects that count).
|
||||
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight);
|
||||
// daemon rejects that count). `margin` widens the window (the first sweep after a load uses a bigger one).
|
||||
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight,
|
||||
int margin = kSentSweepReorgMarginBlocks);
|
||||
// Window margin: steady-state kSentSweepReorgMarginBlocks, bigger on the first sweep after a load.
|
||||
int sentSweepMargin(bool firstAfterLoad);
|
||||
// A: the bottom edge the window covers (W - margin), clamped to >= 1. The anchor whose block hash is stored
|
||||
// (H_A) and re-verified on the next load.
|
||||
int sentSweepAnchor(int fromHeight, bool firstAfterLoad);
|
||||
// A watermark saved long ago makes a huge "windowed" call: past this gap, use callLong and count failures
|
||||
// toward the backoff, so it doesn't time out on every block while the daemon keeps working.
|
||||
bool sentSweepNeedsLongCall(int tip, int fromHeight);
|
||||
// Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h.
|
||||
long fullSentSweepRetryDelaySec(int consecutiveFailures);
|
||||
// Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout,
|
||||
|
||||
@@ -1314,32 +1314,61 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
|
||||
const bool sweepIsFull = sentSweepIsFull(currentBlockHeight, snapshot.sentScanFromHeight);
|
||||
if (!snapshot.shieldedAddresses.empty() && !snapshot.skipSentSweep &&
|
||||
(result.shieldedScanComplete || (snapshot.allowMidPassSentSweep && !sweepIsFull))) {
|
||||
result.sentSweepRan = true;
|
||||
result.sentSweepFull = sweepIsFull;
|
||||
const json params = sentSweepParams(snapshot.shieldedAddresses.front(), currentBlockHeight,
|
||||
snapshot.sentScanFromHeight);
|
||||
const auto sweepStart = std::chrono::steady_clock::now();
|
||||
try {
|
||||
json sent = result.sentSweepFull
|
||||
? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec)
|
||||
: rpc.call("z_listsentbyaddress", params);
|
||||
if (sent.is_array()) {
|
||||
for (const auto& sentTx : sent) {
|
||||
auto txid = readOptional<std::string>(sentTx, "txid");
|
||||
if (!txid || txid->empty()) continue;
|
||||
appendSentTransparentOutputs(result.transactions, *txid, sentTx);
|
||||
knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction
|
||||
}
|
||||
result.sentSweepOk = true;
|
||||
result.sentSweepHeight = currentBlockHeight;
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
transactionRpcError = true;
|
||||
result.sentSweepErrKind = rpc::rpcFailureKindOf(e);
|
||||
result.replyTimedOut |= result.sentSweepErrKind == rpc::RpcFailureKind::ReplyTimeout;
|
||||
DEBUG_LOGF("z_listsentbyaddress error: %s\n", e.what());
|
||||
int effectiveFromHeight = snapshot.sentScanFromHeight;
|
||||
bool effectiveFull = sweepIsFull;
|
||||
// Anchor verify: a watermark restored from disk is trusted only if the block at A still matches the
|
||||
// stored H_A (no reorg deeper than the margin, same chain/datadir). A mismatch, or an inability to
|
||||
// check, drops the watermark and forces a full sweep. Checked only for a windowed sweep.
|
||||
if (snapshot.sentVerifyPending && !sweepIsFull && snapshot.sentVerifyAnchorHeight > 0) {
|
||||
bool anchorOk = false;
|
||||
try {
|
||||
json h = rpc.call("getblockhash", json::array({snapshot.sentVerifyAnchorHeight}));
|
||||
if (h.is_string()) anchorOk = (h.get<std::string>() == snapshot.sentVerifyAnchorHash);
|
||||
} catch (...) { anchorOk = false; }
|
||||
if (!anchorOk) { effectiveFromHeight = -1; effectiveFull = true; }
|
||||
}
|
||||
// A verify that turned a windowed mid-pass sweep into a full one must wait for a completed pass,
|
||||
// rather than run a whole-wallet sweep in the middle of one.
|
||||
if (!(effectiveFull && !result.shieldedScanComplete)) {
|
||||
result.sentSweepRan = true;
|
||||
result.sentSweepFull = effectiveFull;
|
||||
// Bigger margin on the first windowed sweep after a load (absorbs wallet-notifier lag at save).
|
||||
const int margin = sentSweepMargin(snapshot.sentVerifyPending && !effectiveFull);
|
||||
const json params = sentSweepParams(snapshot.shieldedAddresses.front(), currentBlockHeight,
|
||||
effectiveFromHeight, margin);
|
||||
// A watermark saved long ago makes a huge "windowed" call; give it the long timeout too.
|
||||
const bool longCall = effectiveFull || sentSweepNeedsLongCall(currentBlockHeight, effectiveFromHeight);
|
||||
const auto sweepStart = std::chrono::steady_clock::now();
|
||||
try {
|
||||
json sent = longCall
|
||||
? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec)
|
||||
: rpc.call("z_listsentbyaddress", params);
|
||||
if (sent.is_array()) {
|
||||
for (const auto& sentTx : sent) {
|
||||
auto txid = readOptional<std::string>(sentTx, "txid");
|
||||
if (!txid || txid->empty()) continue;
|
||||
appendSentTransparentOutputs(result.transactions, *txid, sentTx);
|
||||
knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction
|
||||
}
|
||||
result.sentSweepOk = true;
|
||||
result.sentSweepHeight = currentBlockHeight;
|
||||
// Anchor for the NEXT load's verify: A = W - steady margin, H_A = getblockhash(A). An
|
||||
// empty H_A (fetch failed) just means the next load can't verify and does a full sweep.
|
||||
const int newAnchor = sentSweepAnchor(currentBlockHeight, false);
|
||||
result.sentSweepAnchorHeight = newAnchor;
|
||||
try {
|
||||
json ha = rpc.call("getblockhash", json::array({newAnchor}));
|
||||
if (ha.is_string()) result.sentSweepAnchorHash = ha.get<std::string>();
|
||||
} catch (...) {}
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
transactionRpcError = true;
|
||||
result.sentSweepErrKind = rpc::rpcFailureKindOf(e);
|
||||
result.replyTimedOut |= result.sentSweepErrKind == rpc::RpcFailureKind::ReplyTimeout;
|
||||
DEBUG_LOGF("z_listsentbyaddress error: %s\n", e.what());
|
||||
}
|
||||
result.sentSweepMs = msSince(sweepStart);
|
||||
}
|
||||
result.sentSweepMs = msSince(sweepStart);
|
||||
}
|
||||
|
||||
for (const auto& txid : snapshot.sendTxids) {
|
||||
|
||||
@@ -254,7 +254,13 @@ public:
|
||||
std::unordered_map<std::string, int> priorityScanFloors;
|
||||
bool skipShieldedScans = false; // a Light cycle (or a probe-less recent poll): no receive scans
|
||||
bool skipSentSweep = false;
|
||||
int sentScanFromHeight = -1; // the sweep's watermark; -1 = sweep the whole wallet
|
||||
int sentScanFromHeight = -1; // the sweep's watermark (W); -1 = sweep the whole wallet
|
||||
// A loaded watermark owes an anchor verify before its first windowed sweep: getblockhash(A) must
|
||||
// still equal H_A (no reorg deeper than the margin). A mismatch forces a full sweep. While pending,
|
||||
// the window also uses the bigger first-after-load margin.
|
||||
bool sentVerifyPending = false;
|
||||
int sentVerifyAnchorHeight = -1; // A
|
||||
std::string sentVerifyAnchorHash; // H_A
|
||||
std::size_t transparentRowHint = 0; // listtransactions rows last time: sizes the first page
|
||||
// Same tip and nothing changed since the rows on screen were fetched: reuse their transparent rows
|
||||
// instead of paging listtransactions again.
|
||||
@@ -291,7 +297,9 @@ public:
|
||||
bool sentSweepRan = false;
|
||||
bool sentSweepOk = false;
|
||||
bool sentSweepFull = false;
|
||||
int sentSweepHeight = -1; // the tip a successful sweep covered
|
||||
int sentSweepHeight = -1; // W: the tip a successful sweep covered
|
||||
int sentSweepAnchorHeight = -1; // A = W - margin, the bottom edge the window covered
|
||||
std::string sentSweepAnchorHash; // H_A = getblockhash(A), fetched after a successful sweep
|
||||
double sentSweepMs = 0.0; // time in the sweep (part of scanMs)
|
||||
rpc::RpcFailureKind sentSweepErrKind = rpc::RpcFailureKind::None;
|
||||
// Txids found but not enriched (over the per-cycle cap, or the view failed transiently): the caller
|
||||
|
||||
Reference in New Issue
Block a user