From 879de37d18a9b295e6259e36b7a9e61cc13bb6bf Mon Sep 17 00:00:00 2001 From: DanS Date: Fri, 9 Oct 2026 11:06:34 -0500 Subject: [PATCH] perf(history): scan shielded history in bounded passes, not every block Every block re-ran z_listreceivedbyaddress for every z-address (O(wallet) each, under cs_main) and a whole-wallet z_listsentbyaddress. On a 40-address wallet that was ~80% of wall-clock. The history now advances in passes: - an address is rescanned when it is stale by a number of blocks scaled to its measured cost (8 to 500), never-scanned and hinted addresses first (a hint: z_listunspent shows a note the history doesn't); - z_listsentbyaddress covers only the blocks since the last good sweep (+100 reorg margin); the whole-wallet sweep runs once per connect, with a 10 min / 1 h / 6 h backoff after real failures; - listtransactions pages are sized from last time, and an unchanged tip reuses the transparent rows; - found-but-unenriched txids carry over as a backlog; only -5 is a definitive "not ours" for z_viewtransaction; - between blocks, a cheap recent poll plus a one-address probe. Duty model for a 40-address wallet at 10 s per scan: ~9% instead of ~80%. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 3 + src/app.cpp | 66 ++- src/app.h | 51 +- src/app_network.cpp | 429 +++++++++++---- src/app_security.cpp | 6 +- src/services/history_scan_policy.cpp | 180 +++++++ src/services/history_scan_policy.h | 135 +++++ src/services/network_refresh_service.cpp | 300 ++++++++--- src/services/network_refresh_service.h | 55 +- tests/test_phase4.cpp | 653 ++++++++++++++++++++++- 10 files changed, 1645 insertions(+), 233 deletions(-) create mode 100644 src/services/history_scan_policy.cpp create mode 100644 src/services/history_scan_policy.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 047c87d..c59a53e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -466,6 +466,7 @@ set(APP_SOURCES src/app_security.cpp src/app_sweep.cpp src/app_wizard.cpp + src/services/history_scan_policy.cpp src/services/network_refresh_service.cpp src/services/note_consolidation.cpp src/services/refresh_scheduler.cpp @@ -613,6 +614,7 @@ endif() set(APP_HEADERS src/app.h + src/services/history_scan_policy.h src/services/network_refresh_service.h src/services/refresh_scheduler.h src/services/wallet_security_controller.h @@ -1148,6 +1150,7 @@ endif() if(BUILD_TESTING) add_executable(ObsidianDragonTests tests/test_phase4.cpp + src/services/history_scan_policy.cpp src/services/network_refresh_service.cpp src/services/note_consolidation.cpp src/services/refresh_scheduler.cpp diff --git a/src/app.cpp b/src/app.cpp index cde9ffa..0689dff 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -1373,7 +1373,7 @@ void App::update() removePendingSendTransactions(successOpids, false); removePendingSendTransactions(failedOrStaleOpids, true); // Transaction confirmed by daemon — force immediate data refresh - transactions_dirty_ = true; + markTransactionsDirty(); addresses_dirty_ = true; last_tx_block_height_ = -1; network_refresh_.markWalletMutationRefresh(); @@ -1417,21 +1417,11 @@ void App::update() // Skip balance/tx/address refresh during warmup — RPC calls fail with -28 — and during a // runtime rescan, when the daemon holds cs_main and these calls would block. if (!state_.warming_up && !runtime_rescan_active_) { - if (network_refresh_.consumeDue(RefreshTimer::Transactions)) { - if (shouldRunWalletTransactionRefresh() && shouldRefreshTransactions()) { - // Throttle the routine new-block full history rescan (z_listreceivedbyaddress — - // O(mapWallet), holds cs_main) by its measured cost, so a large wallet doesn't - // re-scan every block and starve connection near the tip. Bypass when there's an - // explicit need: a dirty set, an in-progress multi-cycle shielded scan (which must - // continue to completion), or an in-flight send — all need fresh data immediately. - const bool txExplicitNeed = transactions_dirty_ || shielded_history_scan_pending_ || - hasTransactionSendProgress() || !send_txids_.empty(); - if (txExplicitNeed || txRefreshDue()) { - refreshTransactionData(); - } - } else if (walletDataPage && shouldRefreshRecentTransactions()) { - refreshRecentTransactionData(); - } + // History: refreshTransactionData() decides what this cycle may scan (services::decideTxCycle): + // wallet-wide scans only as their measured cost allows, except explicit work (the first load, an + // invalidation, a new address); sends and other mutations get their rows through Light cycles. + if (network_refresh_.consumeDue(RefreshTimer::Transactions) && shouldRunWalletTransactionRefresh()) { + refreshTransactionData(); } // 0-conf chat: a DEDICATED ~2.5s mempool scan of the chat address, independent of the page's // (slower, 10–15s) Transactions cadence — so incoming messages surface in a few seconds. The @@ -5263,10 +5253,12 @@ void App::refreshNow() { // Trigger immediate refresh on all categories network_refresh_.markImmediateRefresh(); - transactions_dirty_ = true; // Force transaction list update + markTransactionsDirty(); // Force transaction list update addresses_dirty_ = true; // Force address/balance update last_tx_block_height_ = -1; // Reset tx cache - invalidateShieldedHistoryScanProgress(true); + // A full receive re-scan (explicit, so it runs back to back), but keep the sent sweep's watermark: + // every press would otherwise cost a whole-wallet z_listsentbyaddress. + invalidateShieldedHistoryScanProgress(true, /*keepSentScan=*/true); } void App::skipDaemonOutputBacklog() @@ -5828,7 +5820,7 @@ void App::rescanBlockchain() user_initiated_rescan_ = true; // user-triggered rescan → its completion should toast state_.sync.rescan_progress = 0.0f; state_.sync.rescan_status = decision.status; - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; invalidateShieldedHistoryScanProgress(true); @@ -5882,7 +5874,7 @@ void App::repairWallet() user_initiated_rescan_ = true; // user-triggered repair (implies rescan) → completion should toast state_.sync.rescan_progress = 0.0f; state_.sync.rescan_status = decision.status; - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; invalidateShieldedHistoryScanProgress(true); @@ -7346,9 +7338,14 @@ std::string App::transactionSendProgressText() const std::string App::transactionRefreshProgressText() const { using Job = services::NetworkRefreshService::Job; + if (!rpc_ || !rpc_->isConnected()) return {}; // Lite, or no node: there's no full-node history scan bool running = network_refresh_.jobInProgress(Job::Transactions); bool canRefresh = state_.connected && !state_.warming_up && !state_.isLocked(); - if (!running && !(canRefresh && transactions_dirty_)) return {}; + const auto work = historyScanWork(); + // The first load spans several cycles, and the dirty flag now clears after each: keep the banner up + // between them while addresses are still to scan. + const bool initialLoad = !initial_history_scan_complete_ && work.total > 0; + if (!running && !(canRefresh && (transactions_dirty_ || (initialLoad && work.pending > 0)))) return {}; char buf[128]; @@ -7358,26 +7355,23 @@ std::string App::transactionRefreshProgressText() const return buf; } - // Initial bulk history load: the incremental shielded scan streams older transactions in over - // many refresh cycles, so the first batch can appear long before the history is complete. Keep - // the user informed (with a %) even after rows are on screen, so they know it's still loading. - // Progress = fraction of z-addresses scanned at least once (monotonic). Goes quiet for good once - // the first full scan completes; routine per-block re-scans don't re-trigger it. - if (!initial_history_scan_complete_ && !state_.z_addresses.empty()) { - std::size_t total = state_.z_addresses.size(); - std::size_t scanned = std::min(shielded_history_scan_heights_.size(), total); - int pct = total > 0 ? static_cast((scanned * 100) / total) : 0; + // Initial bulk history load: the shielded receive scans stream older transactions in over several + // refresh cycles, so the first batch can appear long before the history is complete. Keep the user + // informed (with a %) even after rows are on screen, so they know it's still loading. Progress is the + // share of z-addresses this pass has covered. Goes quiet for good once the first pass completes; + // routine re-scans afterwards don't re-trigger it. + if (initialLoad) { + const std::size_t total = work.total; + const std::size_t done = total - std::min(work.pending, total); + int pct = static_cast((done * 100) / total); if (pct > 99) pct = 99; // 100% is the "complete" state, which suppresses the banner snprintf(buf, sizeof(buf), TR("tx_loading_history_progress"), pct); return buf; } - // Once history is on screen, stay quiet for routine background refreshes. The incremental - // shielded-receive scan re-dirties transactions_dirty_ on EVERY new block (each block moves the - // "scanned at tip" bar, invalidating prior per-address scans), so for a wallet with more - // z-addresses than the per-cycle scan budget the flag never clears — which left this banner lit - // indefinitely. New receives still appear live as they're scanned; only the INITIAL load - // (nothing displayed yet) and send enrichment (above) warrant a loading banner. + // Once history is on screen, stay quiet for routine background refreshes: new receives appear live + // as they're scanned; only the INITIAL load (nothing displayed yet) and send enrichment (above) + // warrant a loading banner. if (!state_.transactions.empty()) return {}; if (!running && transactions_dirty_) return TR("tx_loading_queued"); diff --git a/src/app.h b/src/app.h index c8f8e82..69ed6bc 100644 --- a/src/app.h +++ b/src/app.h @@ -26,6 +26,7 @@ #include "daemon/stop_delivery.h" #include "daemon/daemon_busy_detect.h" #include "rpc/rpc_busy_gate.h" +#include "services/history_scan_policy.h" #include "services/network_refresh_service.h" #include "services/wallet_security_controller.h" #include "services/wallet_security_workflow.h" @@ -807,8 +808,13 @@ private: void storeTransactionHistoryCacheIfAvailable(); void wipePendingTransactionHistoryCachePassphrase(); void resetTransactionHistoryCacheSession(); + // The wallet locked: drop the cache key, keep the history and its scan progress (an unlock used to + // force a full re-scan of every address, every auto-lock cycle). + void lockTransactionHistoryCacheSession(); void pruneShieldedHistoryScanProgress(); - void invalidateShieldedHistoryScanProgress(bool persistCache); + // Forget the receive-scan heights so every address is scanned again. keepSentScan leaves the sent + // sweep's watermark (and the enrichment backlog), so a plain refresh doesn't cost a whole-wallet sweep. + void invalidateShieldedHistoryScanProgress(bool persistCache, bool keepSentScan = false); // Auto-balance pool selection: drive the periodic hashrate refresh and apply a // freshly-completed snapshot (weighted-random pick + optional miner restart). @@ -1361,7 +1367,10 @@ private: // addressRefreshDue() / txRefreshDue(); only the routine periodic poll is throttled — explicit // refreshes (tab switch, dirty set, in-flight send) call the refresh directly and bypass this. double last_address_scan_ms_ = 0.0; // measured cost of the last address scan - double last_tx_scan_ms_ = 0.0; // measured cost of the last history scan + double last_tx_scan_ms_ = 0.0; // measured cost of the last wallet-wide history scan + // When that scan ran (wall-clock): the history throttle's clock. Not state_.last_tx_update, which + // every pending-send row edit and probe-less poll also stamps (that pushed the next scan out). + std::int64_t last_tx_scan_at_ = 0; // Auto-clear for secrets copied to the clipboard. Only a hash of the copied secret is kept. std::uint64_t clipboard_secret_hash_ = 0; double clipboard_clear_deadline_ = 0.0; @@ -1488,12 +1497,32 @@ private: int last_tx_block_height_ = -1; // block height at last full tx fetch static constexpr int MAX_VIEWTX_PER_CYCLE = 25; // cap z_viewtransaction calls per refresh std::size_t shielded_history_scan_cursor_ = 0; - bool shielded_history_scan_pending_ = false; // False until the first full shielded-history scan finishes. Drives the History tab's // "Loading older history…" progress so the user knows transactions are still streaming in // after the first batch appears; goes quiet for the routine per-block re-scans afterward. bool initial_history_scan_complete_ = false; std::unordered_map shielded_history_scan_heights_; + // The receive-scan pass over every z-address (see services::ShieldedHistoryPass). + services::ShieldedHistoryPass history_pass_; + std::uint64_t history_scan_epoch_ = 0; // bumped when the scan heights are invalidated + std::uint64_t history_sent_epoch_ = 0; // bumped when the sent sweep's watermark is reset + int history_sent_scan_height_ = -1; // tip of the last successful z_listsentbyaddress; -1 = none + int history_full_sweep_failures_ = 0; // whole-wallet sweeps failed in a row (backoff) + std::time_t history_full_sweep_retry_at_ = 0; + std::unordered_set history_enrich_backlog_; // txids found, not enriched yet + double history_addr_scan_ms_ = 0.0; // per-address z_listreceivedbyaddress cost (estimate) + std::size_t history_transparent_rows_ = 0; // rows the last listtransactions chain returned + int history_transparent_tip_ = -1; // ...and the tip it ran at (same tip: rows can be reused) + std::uint64_t tx_dirty_generation_ = 0; // bumped by markTransactionsDirty() + // Addresses holding notes the history doesn't show yet (from the address scan's z_listunspent): scanned + // first in the next wallet-wide cycle, so a receive or payout shows within one cycle on a large wallet + // instead of when the routine pass reaches its address. Each txid hints once per session after it is + // mined (a mempool hint doesn't use that up, so the row also catches its confirmation). + std::unordered_map history_hint_floors_; + std::unordered_set history_hinted_txids_; + // The z-addresses of the last successful address enumeration this session: a never-scanned address new + // to it (an imported key) resets the sent sweep's watermark, even if the list in between came back blank. + std::unordered_set history_listed_z_; // P4b: z_viewtransaction result cache — avoids re-calling the RPC for // txids we've already enriched. Keyed by txid. @@ -1733,8 +1762,17 @@ private: void refreshData(); // Orchestrator: dispatches per-category refreshes void refreshCoreData(bool allowBalance = true); // Balance + blockchain info (can use fast_worker_) void refreshAddressData(); // Address lists + balances - void refreshTransactionData(); // Transaction list + z_viewtransaction enrichment - void refreshRecentTransactionData(); // Lightweight recent/unconfirmed tx poll + // Transaction history: decides the cycle (services::decideTxCycle) and starts it. Safe to call any + // time: it refuses when the node can't be scanned (not connected, locked, warming up, rescanning, busy). + void refreshTransactionData(); + void startTransactionCycle(bool full, bool routineAllowed); // Full or Light cycle + void startRecentTransactionPoll(bool probe); // between-blocks poll + void markTransactionsDirty() { transactions_dirty_ = true; ++tx_dirty_generation_; } + std::vector shieldedAddressList() const; + services::ShieldedHistoryPass::Work historyScanWork() const; + // From the address scan's z_listunspent: addresses holding received notes the history doesn't show + // yet (no row, or rows still unconfirmed while the note is mined) become priority scans. + void noteHistoryHintsFromUnspent(const std::vector& notes); bool refreshEncryptionState(); // Wallet encryption/lock state void refreshBalance(); // Legacy: balance-only refresh (used by specific callers) void refreshAddresses(); // Legacy: standalone address refresh @@ -1744,13 +1782,12 @@ private: bool effectivelySyncing() const; // syncing, or within the post-sync settle window (hysteresis) bool balanceRefreshDue() const; // adaptive: enough time elapsed given the last balance-scan cost? bool addressRefreshDue() const; // same adaptive back-off for the address scan (z_listunspent) - bool txRefreshDue() const; // same adaptive back-off for the history scan (z_listreceivedbyaddress) + bool txRefreshDue() const; // same adaptive back-off for wallet-wide history scans // Shared duty-cycle rule: a scan may resume only once (lastScanMs / kScanDutyCycle) has elapsed since // lastUpdate, so any single O(mapWallet) scan can occupy at most ~kScanDutyCycle of wall-clock. bool scanRefreshDue(std::int64_t lastUpdate, double lastScanMs) const; bool currentPageNeedsWalletDataRefresh() const; bool shouldRunWalletTransactionRefresh() const; - bool shouldRefreshTransactions() const; bool shouldRefreshRecentTransactions() const; void checkAutoLock(); void checkIdleMining(); diff --git a/src/app_network.cpp b/src/app_network.cpp index 20e7609..0f8074f 100644 --- a/src/app_network.cpp +++ b/src/app_network.cpp @@ -99,8 +99,9 @@ std::string unencryptedTransactionHistoryCacheKey(const std::string& walletIdent class AppRefreshRpcGateway final : public NetworkRefreshService::RefreshRpcGateway { public: - AppRefreshRpcGateway(rpc::RPCClient& rpc, std::string source) - : rpc_(rpc), source_(std::move(source)) {} + // `maintenanceOwner`: the App whose rpc_ this is, for callLong() (see App::MaintenanceCallScope). + AppRefreshRpcGateway(rpc::RPCClient& rpc, std::string source, App* maintenanceOwner = nullptr) + : rpc_(rpc), source_(std::move(source)), owner_(maintenanceOwner) {} json call(const std::string& method, const json& params) override { @@ -108,9 +109,20 @@ public: return rpc_.call(method, params); } + // Holds rpc_'s curl lock for up to timeoutSec: as a maintenance call, a main-thread disconnect + // aborts it rather than waiting for it (App::abortMaintenanceCalls()). + json callLong(const std::string& method, const json& params, long timeoutSec) override + { + std::optional maintenance; + if (owner_) maintenance.emplace(*owner_); + rpc::RPCClient::TraceScope trace(source_); + return rpc_.call(method, params, timeoutSec); + } + private: rpc::RPCClient& rpc_; std::string source_; + App* owner_ = nullptr; }; const char* tracePageName(ui::NavPage page) @@ -150,19 +162,6 @@ std::size_t shieldedReceiveScanBudget(ui::NavPage page) return page == ui::NavPage::History ? 8u : 4u; } -// How far the tip may drift past an address's last shielded scan before we re-scan it. A full pass -// scans ~budget addresses per refresh cycle (≈96 per block of wall time), so a wallet with many -// z-addresses takes several blocks to scan fully. With a strict (tolerance 0) "scanned at tip" -// check, new blocks arriving mid-pass would invalidate already-scanned addresses and the pass would -// never complete — leaving transactions_dirty_ (and its "refreshing history" banner + send-progress -// gate) stuck on forever. Scaling the tolerance with the address count lets the pass complete while -// keeping shielded-receive latency minimal for small wallets; it's capped for pathological sizes. -int shieldedScanTipTolerance(std::size_t shieldedAddressCount) -{ - int t = 2 + static_cast(shieldedAddressCount / 96); - return std::min(t, 50); -} - } // namespace // ============================================================================ @@ -841,6 +840,9 @@ void App::onDisconnected(const std::string& reason) opid_poll_in_progress_ = false; opid_filter_unsupported_ = false; // a reconnect may be to a different daemon last_total_balance_zat_.reset(); + history_transparent_rows_ = 0; // the history's scan state (the pass, epochs and watermark are + history_transparent_tip_ = -1; // reset with the cache session below); cost estimates are kept + last_tx_scan_at_ = 0; ui::ShieldDialog::invalidateScope(); address_validation_cache_dirty_ = true; resetTransactionHistoryCacheSession(); @@ -1043,7 +1045,9 @@ bool App::addressRefreshDue() const bool App::txRefreshDue() const { - return scanRefreshDue(state_.last_tx_update, last_tx_scan_ms_); + // Keyed on the last wallet-wide scan, not state_.last_tx_update (pending-send row edits and probe-less + // polls stamp that too, which kept pushing the next scan out). + return scanRefreshDue(last_tx_scan_at_, last_tx_scan_ms_); } bool App::currentPageNeedsWalletDataRefresh() const @@ -1059,7 +1063,7 @@ bool App::shouldRunWalletTransactionRefresh() const { if (currentPageNeedsWalletDataRefresh()) return true; if (hasTransactionSendProgress() || !send_txids_.empty()) return true; - return transactions_dirty_ && !shielded_history_scan_pending_; + return transactions_dirty_; } void App::setCurrentPage(ui::NavPage page) @@ -1083,7 +1087,7 @@ void App::setCurrentPage(ui::NavPage page) network_refresh_.reset(RefreshTimer::Core); break; case NP::History: - transactions_dirty_ = true; + markTransactionsDirty(); refreshTransactionData(); network_refresh_.reset(RefreshTimer::Transactions); break; @@ -1108,25 +1112,14 @@ void App::setCurrentPage(ui::NavPage page) // Screenshot sweep (tab-only + full-UI) + its demo data live in src/app_sweep.cpp. -bool App::shouldRefreshTransactions() const -{ - // NOTE: this is block-height / dirty driven, NOT interval-gated. It returns true only when a new - // block arrived (currentBlocks != last_tx_block_height_), the history was never fetched, or - // something marked it dirty (tab entry, a send, a reorg, etc.). The Transactions timer only - // controls how often this CHECK runs; between blocks the lightweight recent-poll - // (shouldRefreshRecentTransactions / TxAge) handles mempool + unconfirmed deltas instead. - const int currentBlocks = state_.sync.blocks; - return network_refresh_.shouldRefreshTransactions(last_tx_block_height_, - currentBlocks, - transactions_dirty_); -} - bool App::shouldRefreshRecentTransactions() const { + // No same-tip condition: at a new tip services::decideTxCycle runs a Full or Light cycle first when it + // can, so the cheap poll runs there only while the scan throttle holds the Full cycle back (it keeps + // transparent rows and confirmations live meanwhile, and doesn't consume the new tip). using RefreshTimer = services::NetworkRefreshService::Timer; return network_refresh_.isDue(RefreshTimer::TxAge) && last_tx_block_height_ >= 0 - && state_.sync.blocks == last_tx_block_height_ && !state_.transactions.empty() && !transactions_dirty_ && !addresses_dirty_; @@ -1643,6 +1636,14 @@ void App::resetTransactionHistoryCacheSession() wipePendingTransactionHistoryCachePassphrase(); transaction_history_cache_loaded_ = false; invalidateShieldedHistoryScanProgress(false); + history_listed_z_.clear(); +} + +void App::lockTransactionHistoryCacheSession() +{ + transaction_history_cache_.lockKey(); + wipePendingTransactionHistoryCachePassphrase(); + transaction_history_cache_loaded_ = false; } void App::pruneShieldedHistoryScanProgress() @@ -1662,15 +1663,70 @@ void App::pruneShieldedHistoryScanProgress() } } -void App::invalidateShieldedHistoryScanProgress(bool persistCache) +void App::invalidateShieldedHistoryScanProgress(bool persistCache, bool keepSentScan) { shielded_history_scan_cursor_ = 0; - shielded_history_scan_pending_ = false; initial_history_scan_complete_ = false; // a full re-scan is coming — show load progress again shielded_history_scan_heights_.clear(); + history_pass_.reset(); + ++history_scan_epoch_; // results of scans already in flight no longer apply + history_hint_floors_.clear(); + history_hinted_txids_.clear(); + if (!keepSentScan) { + history_sent_scan_height_ = -1; // the next sweep covers the whole wallet again + history_enrich_backlog_.clear(); + history_full_sweep_failures_ = 0; + history_full_sweep_retry_at_ = 0; + ++history_sent_epoch_; + } if (persistCache) storeTransactionHistoryCacheIfAvailable(); } +std::vector App::shieldedAddressList() const +{ + // The same order as NetworkRefreshService::buildTransactionRefreshSnapshot. + std::vector addresses; + addresses.reserve(state_.z_addresses.size()); + for (const auto& address : state_.z_addresses) { + if (!address.address.empty()) addresses.push_back(address.address); + } + return addresses; +} + +void App::noteHistoryHintsFromUnspent( + const std::vector& notes) +{ + const int tip = state_.sync.blocks; + if (tip <= 0 || notes.empty()) return; + std::unordered_map rowConfirmed; // txid -> any row of it is mined + for (const auto& transaction : state_.transactions) { + if (transaction.txid.empty()) continue; + bool& confirmed = rowConfirmed[transaction.txid]; + confirmed = confirmed || transaction.confirmations >= 1; + } + for (const auto& note : notes) { + if (note.change || note.txid.empty() || note.address.empty()) continue; + if (history_hinted_txids_.count(note.txid)) continue; // once per txid: change/merges can't loop + const auto row = rowConfirmed.find(note.txid); + const bool noRow = row == rowConfirmed.end(); + const bool staleUnconfirmed = !noRow && !row->second && note.confirmations >= 1; + if (!noRow && !staleUnconfirmed) continue; + int& floor = history_hint_floors_[note.address]; + floor = std::max(floor, tip); + // A mempool hint doesn't use up the txid: once mined, its row (still at 0 conf) needs one more scan. + // No loop: an unmined note with a row matches neither case, and a mined one hints once. + if (note.confirmations >= 1) history_hinted_txids_.insert(note.txid); + } +} + +services::ShieldedHistoryPass::Work App::historyScanWork() const +{ + const auto addresses = shieldedAddressList(); + return history_pass_.evaluate(state_.sync.blocks, addresses, shielded_history_scan_heights_, + services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_), + initial_history_scan_complete_); +} + bool App::ensureTransactionHistoryCacheUnlockedFor(const std::string& walletIdentity) { if (walletIdentity.empty()) return false; @@ -1741,7 +1797,7 @@ void App::loadTransactionHistoryCacheIfAvailable() } confirmed_cache_block_ = loaded.tipHeight; transaction_history_cache_loaded_ = true; - transactions_dirty_ = true; + markTransactionsDirty(); network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions); } @@ -1790,15 +1846,12 @@ void App::refreshData() // as each completes, rather than waiting for the slowest phase. refreshCoreData(); - bool addressRefreshNeeded = addresses_dirty_; - bool walletDataPage = currentPageNeedsWalletDataRefresh(); - if (addressRefreshNeeded) + // The address list first: the history needs it (the address callback marks the history due). + // refreshTransactionData() picks the cycle itself (Full / Light / the between-blocks poll / none). + if (addresses_dirty_) refreshAddressData(); - - if (!addressRefreshNeeded && shouldRunWalletTransactionRefresh() && shouldRefreshTransactions()) + else if (shouldRunWalletTransactionRefresh()) refreshTransactionData(); - else if (!addressRefreshNeeded && walletDataPage && shouldRefreshRecentTransactions()) - refreshRecentTransactionData(); // Always refresh peers here — refreshData() only runs on one-shot transitions (connect, // warmup-complete, unlock), so this populates the status-bar peer count immediately on open @@ -1857,7 +1910,7 @@ void App::refreshCoreData(bool allowBalance) resetWitnessRescanProgress(); state_.sync.rescan_progress = 1.0f; // Notes/witnesses were rebuilt — force a fresh history + balance pull. - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; invalidateShieldedHistoryScanProgress(true); if (user_initiated_rescan_) { @@ -2077,15 +2130,17 @@ void App::refreshAddressData() if (!worker_ || !rpc_ || !state_.connected) return; const std::size_t previousAddressCount = state_.z_addresses.size() + state_.t_addresses.size(); const std::string previousWalletIdentity = transactionHistoryCacheWalletIdentity(); + const auto zList = shieldedAddressList(); + std::unordered_set previousZ(zList.begin(), zList.end()); auto addressSnapshot = address_validation_cache_dirty_ ? NetworkRefreshService::AddressRefreshSnapshot{} : NetworkRefreshService::buildAddressRefreshSnapshot(state_); ui::NavPage tracePage = current_page_; - auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Addresses, *worker_, [this, previousAddressCount, previousWalletIdentity, addressSnapshot = std::move(addressSnapshot), tracePage]() -> rpc::RPCWorker::MainCb { + auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Addresses, *worker_, [this, previousAddressCount, previousWalletIdentity, previousZ = std::move(previousZ), addressSnapshot = std::move(addressSnapshot), tracePage]() -> rpc::RPCWorker::MainCb { AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Address refresh")); auto result = NetworkRefreshService::collectAddressRefreshResult(refreshRpc, addressSnapshot); - return [this, previousAddressCount, previousWalletIdentity, result = std::move(result)]() mutable { + return [this, previousAddressCount, previousWalletIdentity, previousZ, result = std::move(result)]() mutable { const bool addrListOk = result.addressListOk; // capture before the move const double scanMs = result.scanMs; // feed the adaptive throttle (addressRefreshDue) if (scanMs > 0.0) last_address_scan_ms_ = scanMs; @@ -2105,16 +2160,44 @@ void App::refreshAddressData() const std::size_t currentAddressCount = state_.z_addresses.size() + state_.t_addresses.size(); const bool addressSetChanged = currentAddressCount != previousAddressCount || transactionHistoryCacheWalletIdentity() != previousWalletIdentity; - if (state_.transactions.empty() || addressSetChanged) { - if (addressSetChanged) { - invalidateShieldedHistoryScanProgress(false); + if (addrListOk) { + // Drop removed addresses' scan heights; the others stay valid (a changed set used to wipe + // every height, so one new address re-scanned the whole wallet). A z-address we never + // scanned that wasn't listed before (a key imported from the Console, say) may bring + // older sends with it, which a windowed sweep can't reach: sweep the whole wallet once. + pruneShieldedHistoryScanProgress(); + // New: in neither the list at dispatch (GUI-created addresses are added to it before the + // refresh, and have no history) nor the last good listing (a blank list in between proves + // nothing), and never scanned. + bool newUnscannedZ = false; + std::unordered_set listedZ; + for (const auto& address : state_.z_addresses) { + if (address.address.empty()) continue; + listedZ.insert(address.address); + if (!previousZ.count(address.address) && !history_listed_z_.count(address.address) && + !shielded_history_scan_heights_.count(address.address)) + newUnscannedZ = true; } - transactions_dirty_ = true; + if (newUnscannedZ && (!previousZ.empty() || !history_listed_z_.empty())) { + history_sent_scan_height_ = -1; + history_full_sweep_failures_ = 0; + history_full_sweep_retry_at_ = 0; + ++history_sent_epoch_; + } + history_listed_z_ = std::move(listedZ); + } + if (state_.transactions.empty() || addressSetChanged) { + markTransactionsDirty(); last_tx_block_height_ = -1; network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions); } if (state_.transactions.empty()) loadTransactionHistoryCacheIfAvailable(); - else storeTransactionHistoryCacheIfAvailable(); + else if (addrListOk) storeTransactionHistoryCacheIfAvailable(); + // Hints compare the notes with the history on screen: only once it's there (the cache loaded + // just above, or a pass is done). Before that every note looks missing, and the first pass scans + // every address anyway. + if (addrListOk && (!state_.transactions.empty() || initial_history_scan_complete_)) + noteHistoryHintsFromUnspent(unspentNotes); maybeFinishTransactionSendProgress(); // Addresses (hence identity + count) are now known — refresh the wallet index's cached // metadata for the active wallet. Throttled: upsert() only writes on a real change. @@ -2130,46 +2213,161 @@ void App::refreshAddressData() void App::refreshTransactionData() { - if (!worker_ || !rpc_ || !state_.connected) return; - if (addresses_dirty_) { - refreshAddressData(); - network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions); + // The one entry point for history refreshes (the periodic timer, connect, tab entry, the address + // callback's re-route), so every caller gets the same rules. Wallet-wide scans hold the daemon's + // cs_main: never while it can't take them. + if (!worker_ || !rpc_ || !rpc_->isConnected() || !state_.connected) return; + if (state_.isLocked() || state_.warming_up || runtime_rescan_active_ || rpc_busy_gate_->busy()) return; + + const int tip = state_.sync.blocks; + const auto work = historyScanWork(); + services::TxCycleInputs in; + in.neverFetched = last_tx_block_height_ < 0; + in.newTip = tip != last_tx_block_height_; + in.dirty = transactions_dirty_; + in.sendTracking = !send_txids_.empty(); + in.explicitScanWork = work.explicitWork; + in.routineScanWork = work.routineWork; + in.gateOpen = txRefreshDue(); + in.syncing = effectivelySyncing(); + in.heightKnown = tip > 0; + in.recentDue = currentPageNeedsWalletDataRefresh() && shouldRefreshRecentTransactions(); + const auto cycle = services::decideTxCycle(in); + switch (cycle) { + case services::TxCycle::None: + return; + case services::TxCycle::Recent: + case services::TxCycle::RecentWithProbe: + startRecentTransactionPoll(cycle == services::TxCycle::RecentWithProbe); + return; + case services::TxCycle::Full: + case services::TxCycle::Light: + if (addresses_dirty_) { + // The history needs the current address list; the address callback marks it due again. + refreshAddressData(); + network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions); + return; + } + startTransactionCycle(cycle == services::TxCycle::Full, in.gateOpen); return; } +} +void App::startTransactionCycle(bool full, bool routineAllowed) +{ const int currentBlocks = state_.sync.blocks; - if (last_tx_block_height_ < 0 || currentBlocks != last_tx_block_height_ || - !shielded_history_scan_pending_) { - shielded_history_scan_cursor_ = 0; - shielded_history_scan_pending_ = false; - } auto transactionSnapshot = NetworkRefreshService::buildTransactionRefreshSnapshot( state_, viewtx_cache_, send_txids_); transactionSnapshot.pendingOpids.insert(pending_opids_.begin(), pending_opids_.end()); if (settings_) transactionSnapshot.miningAddresses = settings_->getMiningAddresses(); transactionSnapshot.shieldedScanHeights = shielded_history_scan_heights_; - transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_; - transactionSnapshot.maxShieldedReceiveScans = shieldedReceiveScanBudget(current_page_); - transactionSnapshot.shieldedScanTipTolerance = - shieldedScanTipTolerance(transactionSnapshot.shieldedAddresses.size()); - // When fully synced, widen the re-scan tolerance so the O(mapWallet) per-address history scan runs - // every several blocks instead of ~every 2 — the heaviest residual synced-state cs_main cost. The - // balance poll still surfaces incoming funds on its own cadence; only the detailed history list lags a - // few minutes. (During catch-up the tx refresh is suppressed by kSyncProfile, so this is synced-only.) - if (!effectivelySyncing()) - transactionSnapshot.shieldedScanTipTolerance = - std::max(transactionSnapshot.shieldedScanTipTolerance, 8); + transactionSnapshot.transparentRowHint = history_transparent_rows_; + transactionSnapshot.reuseTransparentRows = !transactions_dirty_ && currentBlocks > 0 && + history_transparent_tip_ == currentBlocks && !state_.transactions.empty(); + transactionSnapshot.enrichBacklogTxids = history_enrich_backlog_; + if (full) { + const auto& addresses = transactionSnapshot.shieldedAddresses; + const bool wasActive = history_pass_.active(); + history_pass_.begin(currentBlocks, addresses, shielded_history_scan_heights_, + services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_), + initial_history_scan_complete_, routineAllowed); + if (!wasActive && history_pass_.active()) shielded_history_scan_cursor_ = 0; // a new pass: from the top + transactionSnapshot.shieldedScanFloorHeight = history_pass_.scanFloor(currentBlocks, routineAllowed); + transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_; + transactionSnapshot.maxShieldedReceiveScans = shieldedReceiveScanBudget(current_page_); + auto listed = [&](const std::string& address) { + return std::find(addresses.begin(), addresses.end(), address) != addresses.end(); + }; + // Never-scanned addresses first, whatever the cursor (the explicit work: a new or imported + // address costs one scan, not a pass). + for (const auto& address : addresses) { + if (!shielded_history_scan_heights_.count(address)) transactionSnapshot.priorityScanFloors[address] = currentBlocks; + } + if (routineAllowed) { + // Addresses holding notes the history doesn't show yet (noteHistoryHintsFromUnspent). + for (const auto& [address, floor] : history_hint_floors_) { + if (listed(address)) transactionSnapshot.priorityScanFloors.emplace(address, floor); + } + // The chat reply address once per new tip, so messages also surface off the Chat tab. + if (!lite_wallet_ && chat::hushChatFeatureEnabledAtBuild() && chat_service_.hasIdentity() && settings_) { + const std::string chatZ = settings_->getChatReplyZaddr(); + if (!chatZ.empty() && listed(chatZ)) transactionSnapshot.priorityScanFloors[chatZ] = currentBlocks; + } + } + transactionSnapshot.sentScanFromHeight = history_sent_scan_height_; + // A windowed sweep mid-pass too, now and then, so an untracked send (the Console, another wallet on + // the same seed) doesn't wait for a whole pass on a large wallet: at most every quarter of the pass + // cadence, and only in cycles the throttle allows. + const int staleAfter = services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_); + transactionSnapshot.allowMidPassSentSweep = + routineAllowed && history_sent_scan_height_ >= 0 && history_sent_scan_height_ < currentBlocks && + currentBlocks - history_sent_scan_height_ >= std::max(services::kHistoryMinStaleBlocks, staleAfter / 4); + // The sweep runs once per tip, when the throttle allows (or the first load is still running), + // and backs off after whole-wallet sweeps that failed. + transactionSnapshot.skipSentSweep = + (!routineAllowed && initial_history_scan_complete_) || + (history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) || + (history_sent_scan_height_ < 0 && std::time(nullptr) < history_full_sweep_retry_at_); + } else { + // Light: rows now (a send, a mutation), no wallet-wide z-scans. + transactionSnapshot.skipShieldedScans = true; + transactionSnapshot.skipSentSweep = true; + transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_; + } + const std::uint64_t scanEpoch = history_scan_epoch_; + const std::uint64_t sentEpoch = history_sent_epoch_; + const std::uint64_t dirtyGen = tx_dirty_generation_; + const bool snapshotHadZ = !transactionSnapshot.shieldedAddresses.empty(); ui::NavPage tracePage = current_page_; - auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks, - transactionSnapshot = std::move(transactionSnapshot), tracePage]() -> rpc::RPCWorker::MainCb { - AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Transaction refresh")); + auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks, full, + transactionSnapshot = std::move(transactionSnapshot), tracePage, scanEpoch, sentEpoch, dirtyGen, + snapshotHadZ]() -> rpc::RPCWorker::MainCb { + AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, full ? "Transaction refresh" : "Transaction refresh (light)"), + this); auto result = NetworkRefreshService::collectTransactionRefreshResult( refreshRpc, transactionSnapshot, currentBlocks, MAX_VIEWTX_PER_CYCLE); - return [this, result = std::move(result)]() mutable { - bool shieldedScanComplete = result.shieldedScanComplete; - if (result.scanMs > 0.0) last_tx_scan_ms_ = result.scanMs; // feed the adaptive throttle (txRefreshDue) + return [this, full, scanEpoch, sentEpoch, dirtyGen, snapshotHadZ, result = std::move(result)]() mutable { + const std::time_t now = std::time(nullptr); + const int resultTip = result.blockHeight; + const std::size_t attempted = result.shieldedAddressesScanned; + const std::size_t succeeded = result.shieldedScansSucceeded; + const bool sameScanEpoch = scanEpoch == history_scan_epoch_; + // Only history_sent_epoch_ invalidates the sent side (watermark, backoff, backlog): a plain + // refresh keeps the sent scan, so it doesn't throw away an in-flight whole-wallet sweep. + const bool sameSentEpoch = sentEpoch == history_sent_epoch_; + // Feed the throttle (txRefreshDue): Full cycles are the wallet-wide scans it meters. Not the + // whole-wallet sweep, which has its own cadence (once per connect or full invalidation, with a + // backoff): charging up to 120 s of it held routine scans back for 10x that after every connect. + if (full) { + double charged = result.scanMs; + if (result.sentSweepRan && result.sentSweepFull) charged = std::max(1.0, result.scanMs - result.sentSweepMs); + if (charged > 0.0) last_tx_scan_ms_ = charged; + last_tx_scan_at_ = now; + } + if (sameScanEpoch) { + if (succeeded > 0) { + history_addr_scan_ms_ = services::updateScanCostEstimateMs( + history_addr_scan_ms_, result.receiveScanOkMs / static_cast(succeeded)); + } + if (result.transparentChainOk) { + history_transparent_rows_ = result.transparentRowCount; + history_transparent_tip_ = resultTip; + } + } + if (sameSentEpoch) history_enrich_backlog_ = std::move(result.unenrichedTxids); + if (sameSentEpoch) { + if (result.sentSweepOk && result.sentSweepHeight > 0) { + history_sent_scan_height_ = result.sentSweepHeight; + history_full_sweep_failures_ = 0; + history_full_sweep_retry_at_ = 0; + } else if (result.sentSweepRan && result.sentSweepFull && + services::fullSentSweepFailureCounts(result.sentSweepErrKind)) { + ++history_full_sweep_failures_; + history_full_sweep_retry_at_ = now + services::fullSentSweepRetryDelaySec(history_full_sweep_failures_); + } + } std::size_t nextShieldedScanStartIndex = result.nextShieldedScanStartIndex; auto shieldedScanHeights = std::move(result.shieldedScanHeights); NetworkRefreshService::TransactionCacheUpdate cacheUpdate{ @@ -2207,12 +2405,32 @@ void App::refreshTransactionData() NetworkRefreshService::applyTransactionRefreshResult( state_, cacheUpdate, std::move(result), std::time(nullptr)); pruneTrackedSendTxids(); - shielded_history_scan_heights_ = std::move(shieldedScanHeights); + // Pass bookkeeping on the main thread, from the merged heights. Not from a cycle whose address + // list came back blank (enumeration failed): it scanned nothing and knows nothing. + if (full && sameScanEpoch && (snapshotHadZ || shielded_history_scan_heights_.empty())) { + shielded_history_scan_heights_ = std::move(shieldedScanHeights); + shielded_history_scan_cursor_ = nextShieldedScanStartIndex; + const auto addresses = shieldedAddressList(); + // A list blanked by a failed enumeration after dispatch proves nothing: don't end the pass + // or call the first load complete from it. + if (!(snapshotHadZ && addresses.empty())) { + history_pass_.finish(resultTip, addresses, shielded_history_scan_heights_, attempted, succeeded); + for (auto it = history_hint_floors_.begin(); it != history_hint_floors_.end();) { + const auto height = shielded_history_scan_heights_.find(it->first); + if (height != shielded_history_scan_heights_.end() && height->second >= it->second) + it = history_hint_floors_.erase(it); + else + ++it; + } + const auto work = historyScanWork(); + if (work.total > 0 && work.missing == 0 && + !(history_pass_.active() && history_pass_.explicitPass())) + initial_history_scan_complete_ = true; + } + } storeTransactionHistoryCacheIfAvailable(); - shielded_history_scan_cursor_ = nextShieldedScanStartIndex; - shielded_history_scan_pending_ = !shieldedScanComplete; - if (shieldedScanComplete) initial_history_scan_complete_ = true; - transactions_dirty_ = !shieldedScanComplete; + // A mutation that landed while this cycle ran isn't covered by it: stay dirty for it. + if (tx_dirty_generation_ == dirtyGen) transactions_dirty_ = false; maybeFinishTransactionSendProgress(); }; }, 3); @@ -2245,34 +2463,44 @@ void App::pruneTrackedSendTxids() } } -void App::refreshRecentTransactionData() +void App::startRecentTransactionPoll(bool probe) { - if (!worker_ || !rpc_ || !state_.connected) return; - if (!shouldRefreshRecentTransactions()) return; - const int currentBlocks = state_.sync.blocks; auto transactionSnapshot = NetworkRefreshService::buildTransactionRefreshSnapshot( state_, viewtx_cache_, send_txids_); transactionSnapshot.pendingOpids.insert(pending_opids_.begin(), pending_opids_.end()); if (settings_) transactionSnapshot.miningAddresses = settings_->getMiningAddresses(); transactionSnapshot.shieldedScanHeights = shielded_history_scan_heights_; - transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_; + // Between blocks: recent listtransactions rows, plus (when the scan throttle allows) one receive scan + // of the address scanned longest ago. It no longer advances the pass cursor. + transactionSnapshot.skipShieldedScans = !probe; transactionSnapshot.maxShieldedReceiveScans = 1; - transactionSnapshot.shieldedScanTipTolerance = - shieldedScanTipTolerance(transactionSnapshot.shieldedAddresses.size()); - if (!effectivelySyncing()) // synced: widen the re-scan tolerance (see refreshTransactionData) - transactionSnapshot.shieldedScanTipTolerance = - std::max(transactionSnapshot.shieldedScanTipTolerance, 8); + transactionSnapshot.shieldedScanStartIndex = probe + ? services::oldestScannedAddressIndex(transactionSnapshot.shieldedAddresses, shielded_history_scan_heights_) + : 0; + const std::uint64_t scanEpoch = history_scan_epoch_; ui::NavPage tracePage = current_page_; auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks, - transactionSnapshot = std::move(transactionSnapshot), tracePage]() -> rpc::RPCWorker::MainCb { + transactionSnapshot = std::move(transactionSnapshot), tracePage, probe, scanEpoch]() -> rpc::RPCWorker::MainCb { AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Recent transaction poll")); auto result = NetworkRefreshService::collectRecentTransactionRefreshResult( refreshRpc, transactionSnapshot, currentBlocks); - return [this, result = std::move(result)]() mutable { - std::size_t nextShieldedScanStartIndex = result.nextShieldedScanStartIndex; + return [this, probe, scanEpoch, result = std::move(result)]() mutable { + // The poll doesn't consume the tip: newTip stays set until a Full or Light cycle applies at it, + // so the Full cycle still runs once the throttle allows. (Read here, not at dispatch, so a reset + // to -1 meanwhile is kept.) + int pollTip = last_tx_block_height_; + if (probe && result.shieldedAddressesScanned > 0) { + // The probe is a wallet-wide scan: it counts for the throttle. + if (result.scanMs > 0.0) last_tx_scan_ms_ = result.scanMs; + last_tx_scan_at_ = std::time(nullptr); + if (scanEpoch == history_scan_epoch_ && result.shieldedScansSucceeded > 0) { + history_addr_scan_ms_ = services::updateScanCostEstimateMs( + history_addr_scan_ms_, result.receiveScanOkMs / static_cast(result.shieldedScansSucceeded)); + } + } auto shieldedScanHeights = std::move(result.shieldedScanHeights); NetworkRefreshService::TransactionCacheUpdate cacheUpdate{ viewtx_cache_, @@ -2280,7 +2508,7 @@ void App::refreshRecentTransactionData() confirmed_tx_cache_, confirmed_tx_ids_, confirmed_cache_block_, - last_tx_block_height_ + pollTip }; // HushChat: decrypt & thread any harvested chat memos BEFORE `result` is moved (see the // full-refresh path above for rationale; the store dedups across both paths). @@ -2307,8 +2535,7 @@ void App::refreshRecentTransactionData() } NetworkRefreshService::applyTransactionRefreshResult( state_, cacheUpdate, std::move(result), std::time(nullptr)); - shielded_history_scan_heights_ = std::move(shieldedScanHeights); - shielded_history_scan_cursor_ = nextShieldedScanStartIndex; + if (scanEpoch == history_scan_epoch_) shielded_history_scan_heights_ = std::move(shieldedScanHeights); storeTransactionHistoryCacheIfAvailable(); maybeFinishTransactionSendProgress(); }; @@ -2341,7 +2568,7 @@ bool App::refreshEncryptionState() return [this, result]() { NetworkRefreshService::applyWalletEncryptionResult(state_, result); if (state_.isLocked()) { - resetTransactionHistoryCacheSession(); + lockTransactionHistoryCacheSession(); } else if (state_.transactions.empty()) { loadTransactionHistoryCacheIfAvailable(); } else { @@ -5855,7 +6082,7 @@ void App::submitZSendMany(const std::string& from, const std::string& to, double // A send changes address balances — refresh on next cycle addresses_dirty_ = true; // Force transaction list refresh so the sent tx appears immediately - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; requestForcedBalanceRefresh(); // the user's own send must update the balance now, past the throttle network_refresh_.markWalletMutationRefresh(); @@ -6030,7 +6257,7 @@ void App::runtimeRescan(int startHeight) user_initiated_rescan_ = true; // user clicked Rescan → completion should toast state_.sync.rescan_progress = 0.0f; state_.sync.rescan_status = "Rescanning from block " + std::to_string(startHeight) + "..."; - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; invalidateShieldedHistoryScanProgress(true); @@ -6052,7 +6279,7 @@ void App::runtimeRescan(int startHeight) resetWitnessRescanProgress(); if (ok) { state_.sync.rescan_progress = 1.0f; - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; invalidateShieldedHistoryScanProgress(true); ui::Notifications::instance().success("Blockchain rescan complete"); diff --git a/src/app_security.cpp b/src/app_security.cpp index e68800c..1ac0ef4 100644 --- a/src/app_security.cpp +++ b/src/app_security.cpp @@ -547,7 +547,7 @@ void App::lockWallet() { if (ok) { state_.locked = true; state_.unlocked_until = 0; - resetTransactionHistoryCacheSession(); + lockTransactionHistoryCacheSession(); // keeps the history's scan progress lock_failure_warned_ = false; DEBUG_LOGF("[App] Wallet locked\n"); } else { @@ -642,7 +642,7 @@ void App::refreshWalletEncryptionState() { settings_->save(); } if (state_.locked) { - resetTransactionHistoryCacheSession(); + lockTransactionHistoryCacheSession(); } else if (state_.transactions.empty()) { loadTransactionHistoryCacheIfAvailable(); } else { @@ -1799,7 +1799,7 @@ void App::renderDecryptWalletDialog() { // Force address + peer refresh invalidateAddressValidationCache(); addresses_dirty_ = true; - transactions_dirty_ = true; + markTransactionsDirty(); last_tx_block_height_ = -1; refreshWalletEncryptionState(); diff --git a/src/services/history_scan_policy.cpp b/src/services/history_scan_policy.cpp new file mode 100644 index 0000000..15cb926 --- /dev/null +++ b/src/services/history_scan_policy.cpp @@ -0,0 +1,180 @@ +// DragonX Wallet - ImGui Edition +// Copyright 2024-2026 The Hush Developers +// Released under the GPLv3 + +#include "history_scan_policy.h" + +#include +#include + +namespace dragonx { +namespace services { + +int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs) +{ + if (addressCount == 0 || !(perAddressScanMs > 0.0)) return kHistoryMinStaleBlocks; + const double blocks = std::ceil((static_cast(addressCount) * perAddressScanMs / 1000.0) / + (kHistoryBlockSeconds * kHistoryScanDuty)); + if (!(blocks < kHistoryMaxStaleBlocks)) return kHistoryMaxStaleBlocks; // also NaN / inf + return std::max(kHistoryMinStaleBlocks, static_cast(blocks)); +} + +double updateScanCostEstimateMs(double previousMs, double sampleMs) +{ + if (!(sampleMs > 0.0)) return previousMs; + if (!(previousMs > 0.0) || sampleMs > previousMs) return sampleMs; + return 0.75 * previousMs + 0.25 * sampleMs; +} + +int transparentPageSize(std::size_t previousRowCount) +{ + const std::size_t want = previousRowCount + std::max(200, previousRowCount / 10); + return static_cast(std::clamp(want, kTransparentPageMin, kTransparentPageMax)); +} + +bool sentSweepIsFull(int tip, int fromHeight) +{ + return tip <= 0 || fromHeight < 0 || fromHeight > tip; +} + +nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight) +{ + nlohmann::json params = nlohmann::json::array(); + params.push_back(anyZAddress); + params.push_back(0); + if (sentSweepIsFull(tip, fromHeight)) return params; + const int maxDepth = (tip - fromHeight) + kSentSweepReorgMarginBlocks; + params.push_back(2); // filter type 2: depth + params.push_back(maxDepth); // at most this deep (mempool txs, depth 0, always pass) + return params; +} + +long fullSentSweepRetryDelaySec(int consecutiveFailures) +{ + if (consecutiveFailures <= 0) return 0; + if (consecutiveFailures == 1) return 600; + if (consecutiveFailures == 2) return 3600; + return 21600; +} + +bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind) +{ + using K = rpc::RpcFailureKind; + return kind == K::ReplyTimeout || kind == K::AppError || kind == K::Other; +} + +std::size_t oldestScannedAddressIndex(const std::vector& addresses, + const std::unordered_map& heights) +{ + std::size_t best = 0; + int bestHeight = 0; + bool haveBest = false; + for (std::size_t i = 0; i < addresses.size(); ++i) { + const auto it = heights.find(addresses[i]); + if (it == heights.end()) return i; // never scanned: first + if (!haveBest || it->second < bestHeight) { + best = i; + bestHeight = it->second; + haveBest = true; + } + } + return best; +} + +int ShieldedHistoryPass::floorAt(int tip) const +{ + return tip > 0 ? std::min(startHeight_, tip) : startHeight_; +} + +ShieldedHistoryPass::Work ShieldedHistoryPass::evaluate(int tip, const std::vector& addresses, + const std::unordered_map& heights, + int staleAfter, bool initialComplete) const +{ + Work work; + work.total = addresses.size(); + const int floor = active_ ? floorAt(tip) : -1; + std::size_t below = 0; + bool staleDue = false; + for (const auto& address : addresses) { + const auto it = heights.find(address); + if (it == heights.end()) { + ++work.missing; + continue; + } + if (active_) { + if (it->second < floor) ++below; + } else if (tip > 0 && it->second < tip - staleAfter) { + staleDue = true; + } + } + work.pending = work.missing + (active_ ? below : (staleDue ? work.total - work.missing : 0)); + work.explicitWork = !stalled() && + (work.missing > 0 || (active_ ? (explicit_ && below > 0) : (staleDue && !initialComplete))); + work.routineWork = !work.explicitWork && work.pending > 0; + return work; +} + +void ShieldedHistoryPass::begin(int tip, const std::vector& addresses, + const std::unordered_map& heights, int staleAfter, + bool initialComplete, bool routineAllowed) +{ + if (active_) { + if (tip > 0 && startHeight_ > tip) startHeight_ = tip; // the chain reorganised below the floor + return; + } + if (tip <= 0 || !(routineAllowed || !initialComplete)) return; + bool staleDue = false; + for (const auto& address : addresses) { + const auto it = heights.find(address); + if (it != heights.end() && it->second < tip - staleAfter) { + staleDue = true; + break; + } + } + if (!staleDue) return; + active_ = true; + explicit_ = !initialComplete; + startHeight_ = tip; +} + +int ShieldedHistoryPass::scanFloor(int tip, bool routineAllowed) const +{ + return (active_ && (explicit_ || routineAllowed)) ? floorAt(tip) : -1; +} + +void ShieldedHistoryPass::finish(int tip, const std::vector& addresses, + const std::unordered_map& heights, std::size_t attempted, + std::size_t succeeded) +{ + if (attempted > 0) stalledCycles_ = succeeded > 0 ? 0 : stalledCycles_ + 1; + // An empty list proves nothing (a failed enumeration blanks it); a real wallet change resets the pass. + if (!active_ || addresses.empty()) return; + const int floor = floorAt(tip); + for (const auto& address : addresses) { + const auto it = heights.find(address); + if (it == heights.end() || it->second < floor) return; // still work left in this pass + } + active_ = false; + explicit_ = false; +} + +void ShieldedHistoryPass::reset() +{ + *this = ShieldedHistoryPass{}; +} + +TxCycle decideTxCycle(const TxCycleInputs& in) +{ + const bool wanted = in.neverFetched || in.newTip || in.dirty || in.explicitScanWork || in.routineScanWork; + const bool scansAllowed = !in.syncing && in.heightKnown; + if (wanted) { + if (scansAllowed && (in.explicitScanWork || in.gateOpen)) return TxCycle::Full; + // Rows now (a mutation, a tracked send at a new tip); wallet-wide scans wait for the throttle. + if (in.neverFetched || in.dirty || (in.sendTracking && in.newTip)) return TxCycle::Light; + } + if (in.recentDue) return (scansAllowed && in.gateOpen) ? TxCycle::RecentWithProbe : TxCycle::Recent; + return TxCycle::None; +} + +} // namespace services +} // namespace dragonx diff --git a/src/services/history_scan_policy.h b/src/services/history_scan_policy.h new file mode 100644 index 0000000..5789df1 --- /dev/null +++ b/src/services/history_scan_policy.h @@ -0,0 +1,135 @@ +// DragonX Wallet - ImGui Edition +// Copyright 2024-2026 The Hush Developers +// Released under the GPLv3 + +#pragma once + +// When the transaction history refresh may scan what. The history comes from three wallet-wide RPCs, each +// holding the daemon's cs_main for its whole run (seconds per call on a large wallet): listtransactions, +// z_listreceivedbyaddress (once per z-address) and z_listsentbyaddress. This decides, per refresh cycle, +// which of them run, so the cost stays bounded and every pass over the z-addresses finishes. Pure and +// main-thread only; App::refreshTransactionData() feeds it and the test target drives it directly. + +#include "rpc/rpc_failure.h" + +#include + +#include +#include +#include +#include + +namespace dragonx { +namespace services { + +constexpr int kHistoryBlockSeconds = 36; // -ac_blocktime=36 +constexpr double kHistoryScanDuty = 0.10; // the same 10% as the other scan throttles +constexpr int kHistoryMinStaleBlocks = 8; // re-scan an address at most every 8 blocks... +constexpr int kHistoryMaxStaleBlocks = 500; // ...and at least every 500 (a safety net) +constexpr int kSentSweepReorgMarginBlocks = 100; // > MAX_REORG_LENGTH (99), plus tip drift +constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish +constexpr int kTransparentPageMin = 1000; +constexpr int kTransparentPageMax = 5000; +constexpr int kStallCyclesBeforeGating = 1; + +// Blocks after which an address's last receive scan counts as stale and a new routine pass is due: the +// pass then costs about kHistoryScanDuty of wall-clock (one z_listreceivedbyaddress of perAddressScanMs per +// address, spread over that many blocks), within [kHistoryMinStaleBlocks, kHistoryMaxStaleBlocks]. +int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs); + +// Running cost estimate: a higher sample replaces it at once, a lower one pulls it down 25%. A sample of 0 +// or less (nothing measured) leaves it as it is. +double updateScanCostEstimateMs(double previousMs, double sampleMs); + +// listtransactions rows to ask for in the first page: last time's count plus headroom, so most wallets +// get every row in one call instead of re-paging the whole history in 1000-row pages. +int transparentPageSize(std::size_t previousRowCount); + +// z_listsentbyaddress: the whole wallet when there's no usable watermark (none yet, or it is above the +// tip after a reorg), else only txs at most (tip - fromHeight) + kSentSweepReorgMarginBlocks deep. +bool sentSweepIsFull(int tip, int fromHeight); +// [z, 0] for the full sweep, [z, 0, 2, maxDepth] for the window: JSON integers, and never 3 params (the +// daemon rejects that count). +nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight); +// Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h. +long fullSentSweepRetryDelaySec(int consecutiveFailures); +// Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout, +// an error answer, anything unclassified), not when the call never ran (held back, aborted, not listening). +bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind); + +// The address scanned longest ago (never-scanned first), for the between-blocks probe. +std::size_t oldestScannedAddressIndex(const std::vector& addresses, + const std::unordered_map& heights); + +// One pass of receive scans over every z-address. An address is done once it was scanned at or above the +// pass's start height (its floor), so a pass ends after ceil(A / budget) cycles however the tip moves. An +// explicit pass (the first load of a wallet, or after an invalidation) may run back to back; a routine +// re-pass (addresses gone stale) runs only as the scan throttle allows. +class ShieldedHistoryPass { +public: + struct Work { + std::size_t total = 0; // z-addresses + std::size_t missing = 0; // never scanned + std::size_t pending = 0; // still to scan in this pass (or in the pass that is due) + bool explicitWork = false; // may skip the scan throttle + bool routineWork = false; // waits for the scan throttle + }; + + Work evaluate(int tip, const std::vector& addresses, + const std::unordered_map& heights, int staleAfter, bool initialComplete) const; + // Start a pass if one is due and allowed (routine passes need routineAllowed, unless the first load + // isn't complete yet). Clamps an active pass's floor after a reorg below it. + void begin(int tip, const std::vector& addresses, const std::unordered_map& heights, + int staleAfter, bool initialComplete, bool routineAllowed); + // The floor a cycle scans up to: -1 means only never-scanned (and priority) addresses, which is all an + // explicit-only cycle may do while a routine pass is active. + int scanFloor(int tip, bool routineAllowed) const; + // After a cycle's heights are merged: count stalls (scans attempted, none succeeded), end the pass once + // no address is missing or below its floor. + void finish(int tip, const std::vector& addresses, const std::unordered_map& heights, + std::size_t attempted, std::size_t succeeded); + void reset(); + + bool active() const { return active_; } + bool explicitPass() const { return explicit_; } + int startHeight() const { return startHeight_; } + // Scans keep failing: explicit work is treated as routine (gated) until one succeeds again. + bool stalled() const { return stalledCycles_ >= kStallCyclesBeforeGating; } + +private: + int floorAt(int tip) const; + + bool active_ = false; + bool explicit_ = false; + int startHeight_ = -1; + int stalledCycles_ = 0; +}; + +enum class TxCycle { + None, + Full, // listtransactions, receive scans, the sent sweep, enrichment + Light, // listtransactions and enrichment only: no wallet-wide z-scans + Recent, // the between-blocks poll: recent listtransactions rows + RecentWithProbe, // ...plus one receive scan of the address scanned longest ago +}; + +struct TxCycleInputs { + bool neverFetched = false; // no history loaded yet this session + bool newTip = false; // the tip moved since the last history apply + bool dirty = false; // something changed the wallet (a send, an import, ...) + bool sendTracking = false; // tracked sends still waiting for their view + bool explicitScanWork = false; // ShieldedHistoryPass::Work::explicitWork + bool routineScanWork = false; // ShieldedHistoryPass::Work::routineWork + bool gateOpen = false; // the scan throttle allows a wallet-wide scan now + bool syncing = false; // the node is catching up (or settling after it) + bool heightKnown = false; // the tip height is known + bool recentDue = false; // the between-blocks poll is due on a wallet page +}; + +// Only explicit work skips the throttle with scans; sends and other mutations get rows now through Light +// cycles; nothing scans while syncing or before the tip is known. Tracking alone needs a new tip for a +// Light cycle, so it can't loop them at one tip. +TxCycle decideTxCycle(const TxCycleInputs& in); + +} // namespace services +} // namespace dragonx diff --git a/src/services/network_refresh_service.cpp b/src/services/network_refresh_service.cpp index be5aaf6..7d7c5df 100644 --- a/src/services/network_refresh_service.cpp +++ b/src/services/network_refresh_service.cpp @@ -1,4 +1,5 @@ #include "network_refresh_service.h" +#include "history_scan_policy.h" #include "rpc/rpc_busy_gate.h" #include "../util/logger.h" @@ -134,20 +135,43 @@ void appendTrackedSendPlaceholder(std::vector& transactions, transactions.push_back(std::move(info)); } +// A row the shielded scans (or z_viewtransaction) produce rather than listtransactions. +bool isShieldedRelatedRow(const TransactionInfo& transaction) +{ + return (!transaction.address.empty() && transaction.address[0] == 'z') || + (!transaction.from_address.empty() && transaction.from_address[0] == 'z') || + !transaction.memo.empty(); +} + void appendMissingPreviousTransactions(std::vector& transactions, const std::vector& previousTransactions, bool includeAll, const std::unordered_set& pinnedTxids) { + // Presence is decided against the rows built this cycle, before any is carried: deduping against rows + // just appended kept only the first leg of a multi-recipient send. + std::set> present; + for (const auto& transaction : transactions) present.emplace(transaction.txid, transaction.type); for (const auto& transaction : previousTransactions) { bool pinned = pinnedTxids.find(transaction.txid) != pinnedTxids.end(); - if (!includeAll && !pinned) { - bool shieldedRelated = (!transaction.address.empty() && transaction.address[0] == 'z') || - (!transaction.from_address.empty() && transaction.from_address[0] == 'z') || - !transaction.memo.empty(); - if (!shieldedRelated) continue; - } - if (hasTransactionType(transactions, transaction.txid, transaction.type)) continue; + if (!includeAll && !pinned && !isShieldedRelatedRow(transaction)) continue; + if (present.count({transaction.txid, transaction.type})) continue; + transactions.push_back(transaction); + } +} + +// On a full rebuild, keep the stored rows of sends still waiting to be re-enriched. Presence is decided +// against the rows built this cycle, before any is carried, so every row of a multi-recipient send is kept +// (deduping against rows just appended would keep only the first). +void appendPinnedPreviousTransactions(std::vector& transactions, + const std::vector& previousTransactions, + const std::unordered_set& pinnedTxids) +{ + std::set> present; + for (const auto& transaction : transactions) present.emplace(transaction.txid, transaction.type); + for (const auto& transaction : previousTransactions) { + if (!pinnedTxids.count(transaction.txid)) continue; + if (present.count({transaction.txid, transaction.type})) continue; transactions.push_back(transaction); } } @@ -767,6 +791,9 @@ NetworkRefreshService::AddressRefreshResult NetworkRefreshService::collectAddres lite.confirmations = (nz.contains("rawconfirmations") && nz["rawconfirmations"].is_number_integer()) ? nz["rawconfirmations"].get() : nz.value("confirmations", 0); + if (nz.contains("address") && nz["address"].is_string()) lite.address = nz["address"].get(); + if (nz.contains("txid") && nz["txid"].is_string()) lite.txid = nz["txid"].get(); + if (nz.contains("change") && nz["change"].is_boolean()) lite.change = nz["change"].get(); result.unspentNotes.push_back(lite); } } @@ -1044,10 +1071,13 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr result.blockHeight = currentBlockHeight; result.shieldedAddressCount = snapshot.shieldedAddresses.size(); result.shieldedScanHeights = snapshot.shieldedScanHeights; - // Time the whole scan — the per-address z_listreceivedbyaddress pass is O(mapWallet) and holds the - // daemon's cs_main. The measured cost feeds the caller's adaptive throttle (txRefreshDue()) so the - // routine full history rescan can't starve block connection on a large wallet. + // Time the whole cycle: listtransactions, the per-address z_listreceivedbyaddress scans and + // z_listsentbyaddress are each O(mapWallet) and hold the daemon's cs_main. The measured cost feeds the + // caller's throttle (txRefreshDue()) so routine scans can't starve block connection on a large wallet. const auto scanStart = std::chrono::steady_clock::now(); + auto msSince = [](std::chrono::steady_clock::time_point from) { + return std::chrono::duration(std::chrono::steady_clock::now() - from).count(); + }; std::set knownTxids; HushChatMemoOutputMap hushChatReceivedOutputs; @@ -1056,47 +1086,62 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr : nullptr; bool transactionRpcError = false; - try { - constexpr int pageSize = 1000; - int skip = 0; - while (true) { - json transactions = rpc.call("listtransactions", json::array({"", pageSize, skip})); - appendTransparentTransactions(result.transactions, knownTxids, transactions, snapshot.miningAddresses); - if (!transactions.is_array() || transactions.size() < static_cast(pageSize)) break; - skip += pageSize; + + // ── Transparent rows (listtransactions) ───────────────────────────────────────────────────────── + if (snapshot.reuseTransparentRows && !snapshot.previousTransactions.empty()) { + // Same tip and nothing changed since the rows on screen were fetched: keep their transparent rows + // instead of paging the whole history again. Shielded rows come from the scans below, and rows of + // tracked sends and pending ops are rebuilt by enrichment. + for (const auto& transaction : snapshot.previousTransactions) { + if (isShieldedRelatedRow(transaction)) continue; + if (snapshot.sendTxids.count(transaction.txid) || snapshot.pendingOpids.count(transaction.txid)) continue; + result.transactions.push_back(transaction); + if (!transaction.txid.empty()) knownTxids.insert(transaction.txid); + } + result.transparentRowsReused = true; + } else { + try { + // The first page holds last time's count plus headroom, so most wallets get every row in one + // call (re-paging 1000 rows at a time formats ~rows^2/2000 rows); skip-paging stays the fallback. + const int pageSize = transparentPageSize(snapshot.transparentRowHint); + for (int skip = 0;; skip += pageSize) { + json page = rpc.call("listtransactions", json::array({"", pageSize, skip})); + appendTransparentTransactions(result.transactions, knownTxids, page, snapshot.miningAddresses); + const std::size_t got = page.is_array() ? page.size() : 0; + result.transparentRowCount += got; + if (got < static_cast(pageSize)) break; + } + result.transparentChainOk = true; + } catch (const std::exception& e) { + transactionRpcError = true; + DEBUG_LOGF("listtransactions error: %s\n", e.what()); } - } catch (const std::exception& e) { - transactionRpcError = true; - DEBUG_LOGF("listtransactions error: %s\n", e.what()); } - // An address counts as "scanned at tip" if its last scan is within shieldedScanTipTolerance - // blocks of the current tip. Tolerance 0 means strict (exact tip); a small tolerance lets a - // multi-cycle pass over many addresses complete even as the tip advances during it. - const int scanStaleThreshold = currentBlockHeight - std::max(0, snapshot.shieldedScanTipTolerance); - auto scannedAtTip = [&](const std::string& address) { - if (currentBlockHeight < 0) return false; - auto it = result.shieldedScanHeights.find(address); - return it != result.shieldedScanHeights.end() && it->second >= scanStaleThreshold; + // ── Shielded receives (z_listreceivedbyaddress, one per z-address) ────────────────────────────── + // A pass re-scans each address once: one scanned below the pass floor (or never) needs a scan, so a + // pass over many addresses completes however the tip moves meanwhile. Priority addresses go first. + const int scanFloor = snapshot.shieldedScanFloorHeight; + auto needsPassScan = [&](const std::string& address) { + const auto it = result.shieldedScanHeights.find(address); + if (it == result.shieldedScanHeights.end()) return true; + return scanFloor >= 0 && it->second < scanFloor; }; - - std::size_t shieldedStart = snapshot.shieldedScanStartIndex; - if (shieldedStart >= snapshot.shieldedAddresses.size()) shieldedStart = 0; - std::size_t nextSearchIndex = shieldedStart; - - for (std::size_t index = shieldedStart; index < snapshot.shieldedAddresses.size(); ++index) { - const auto& address = snapshot.shieldedAddresses[index]; - if (scannedAtTip(address)) { - nextSearchIndex = index + 1; - continue; - } - if (snapshot.maxShieldedReceiveScans > 0 && - result.shieldedAddressesScanned >= snapshot.maxShieldedReceiveScans) { - nextSearchIndex = index; - break; - } + auto needsPriorityScan = [&](const std::string& address) { + const auto priority = snapshot.priorityScanFloors.find(address); + if (priority == snapshot.priorityScanFloors.end()) return false; + const auto it = result.shieldedScanHeights.find(address); + return it == result.shieldedScanHeights.end() || it->second < priority->second; + }; + auto budgetLeft = [&]() { + return snapshot.maxShieldedReceiveScans == 0 || + result.shieldedAddressesScanned < snapshot.maxShieldedReceiveScans; + }; + std::unordered_set scannedNow; + auto scanAddress = [&](const std::string& address) { ++result.shieldedAddressesScanned; - nextSearchIndex = index + 1; + scannedNow.insert(address); + const auto callStart = std::chrono::steady_clock::now(); try { json received = rpc.call("z_listreceivedbyaddress", json::array({address, 0})); appendShieldedReceivedTransactions(result.transactions, @@ -1105,11 +1150,36 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr received, snapshot.miningAddresses, hushChatReceivedOutputsPtr); + ++result.shieldedScansSucceeded; + result.receiveScanOkMs += msSince(callStart); if (currentBlockHeight >= 0) result.shieldedScanHeights[address] = currentBlockHeight; } catch (const std::exception& e) { transactionRpcError = true; DEBUG_LOGF("z_listreceivedbyaddress error: %s\n", e.what()); } + }; + + std::size_t shieldedStart = snapshot.shieldedScanStartIndex; + if (shieldedStart >= snapshot.shieldedAddresses.size()) shieldedStart = 0; + std::size_t nextSearchIndex = shieldedStart; + if (!snapshot.skipShieldedScans) { + for (const auto& address : snapshot.shieldedAddresses) { + if (!budgetLeft()) break; + if (needsPriorityScan(address)) scanAddress(address); + } + for (std::size_t index = shieldedStart; index < snapshot.shieldedAddresses.size(); ++index) { + const auto& address = snapshot.shieldedAddresses[index]; + if (!needsPassScan(address) || scannedNow.count(address)) { + nextSearchIndex = index + 1; + continue; + } + if (!budgetLeft()) { + nextSearchIndex = index; + break; + } + nextSearchIndex = index + 1; + scanAddress(address); + } } auto findPendingShieldedIndex = [&]() -> std::optional { @@ -1117,7 +1187,7 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr std::size_t start = nextSearchIndex < snapshot.shieldedAddresses.size() ? nextSearchIndex : 0; for (std::size_t offset = 0; offset < snapshot.shieldedAddresses.size(); ++offset) { std::size_t index = (start + offset) % snapshot.shieldedAddresses.size(); - if (!scannedAtTip(snapshot.shieldedAddresses[index])) return index; + if (needsPassScan(snapshot.shieldedAddresses[index])) return index; } return std::nullopt; }; @@ -1126,11 +1196,11 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr result.shieldedScanComplete = !pendingShieldedIndex.has_value(); result.nextShieldedScanStartIndex = pendingShieldedIndex.value_or(0); - // Discover outgoing shielded (z->z) sends. A pure z->z send moves no value in or out of the - // transparent pool (valueBalance == 0), so the daemon's `listtransactions` never reports it, - // and — being a send rather than a receive — it never shows up in `z_listreceivedbyaddress` - // either. Its txid therefore never reaches `knownTxids`, so the z_viewtransaction enrichment - // below is never run for it and the send is invisible in History (the reported bug). + // ── Outgoing shielded sends (z_listsentbyaddress) ─────────────────────────────────────────────── + // A pure z->z send moves no value in or out of the transparent pool (valueBalance == 0), so the + // daemon's `listtransactions` never reports it, and — being a send rather than a receive — it never + // shows up in `z_listreceivedbyaddress` either. Its txid only reaches `knownTxids`, and so the + // z_viewtransaction enrichment below, through this call. // // `z_listsentbyaddress` scans the whole wallet for sends — the address argument is NOT used to // filter (the daemon iterates every wallet tx against "*"), so a single call yields all send @@ -1144,13 +1214,22 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr // deduped (by txid+address+amount) against listtransactions-derived t->t sends above, so a // purely-transparent (t->t) send — which the payload ALSO reports — is not double-counted. // - // Gated on shieldedScanComplete so it adds no cost during the multi-cycle initial received - // scan: it is one more O(mapWallet) pass holding cs_main (comparable to the listtransactions - // pass), and the caller's adaptive throttle (txRefreshDue) already accounts for it via scanMs. - if (result.shieldedScanComplete && !snapshot.shieldedAddresses.empty()) { + // It trial-decrypts every send it returns under cs_main, the heaviest call here on a wallet with many + // sends. So it runs once a pass is complete and, past a successful sweep (the caller's watermark), only + // over the txs at most a little deeper than the blocks since then (filter type 2). The whole-wallet + // sweep, needed when there's no watermark, gets a long timeout so it can finish on a large wallet. + const bool sweepIsFull = sentSweepIsFull(currentBlockHeight, snapshot.sentScanFromHeight); + if (!snapshot.shieldedAddresses.empty() && !snapshot.skipSentSweep && + (result.shieldedScanComplete || (snapshot.allowMidPassSentSweep && !sweepIsFull))) { + result.sentSweepRan = true; + result.sentSweepFull = sweepIsFull; + const json params = sentSweepParams(snapshot.shieldedAddresses.front(), currentBlockHeight, + snapshot.sentScanFromHeight); + const auto sweepStart = std::chrono::steady_clock::now(); try { - json sent = rpc.call("z_listsentbyaddress", - json::array({snapshot.shieldedAddresses.front(), 0})); + json sent = result.sentSweepFull + ? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec) + : rpc.call("z_listsentbyaddress", params); if (sent.is_array()) { for (const auto& sentTx : sent) { auto txid = readOptional(sentTx, "txid"); @@ -1158,11 +1237,15 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr appendSentTransparentOutputs(result.transactions, *txid, sentTx); knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction } + result.sentSweepOk = true; + result.sentSweepHeight = currentBlockHeight; } } catch (const std::exception& e) { transactionRpcError = true; + result.sentSweepErrKind = rpc::rpcFailureKindOf(e); DEBUG_LOGF("z_listsentbyaddress error: %s\n", e.what()); } + result.sentSweepMs = msSince(sweepStart); } for (const auto& txid : snapshot.sendTxids) { @@ -1173,17 +1256,41 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr if (!cachedView.outgoing_outputs.empty()) knownTxids.insert(txid); } + // Shielded send rows from an earlier session (the history cache) whose view isn't cached: they count as + // enriched by their age, so nothing would rebuild them, and a windowed sweep never rediscovers an old + // send. Re-enrich them, and keep their stored rows ("pinned") until the daemon has answered. + std::unordered_set rowBacklog; + // A settled stored row's block time and confirmations (>= 10, the History's "confirmed"): reused for + // its re-enrichment instead of a gettransaction per row. An unsettled one still asks, or its stale + // count would stick for the session (a cached view keeps the metadata it was built with). + std::unordered_map> settledStoredMeta; + for (const auto& transaction : snapshot.previousTransactions) { + if (transaction.type != "send" || transaction.txid.empty() || !isShieldedRelatedRow(transaction)) continue; + if (snapshot.viewTxCache.count(transaction.txid) || snapshot.sendTxids.count(transaction.txid) || + snapshot.pendingOpids.count(transaction.txid)) continue; + rowBacklog.insert(transaction.txid); + if (transaction.timestamp != 0 && transaction.confirmations >= 10) + settledStoredMeta.emplace(transaction.txid, std::make_pair(transaction.timestamp, transaction.confirmations)); + } + std::set backlog(snapshot.enrichBacklogTxids.begin(), snapshot.enrichBacklogTxids.end()); + backlog.insert(rowBacklog.begin(), rowBacklog.end()); + std::unordered_set pinnedRows = rowBacklog; + // Tracked sends go first so they get the per-cycle z_viewtransaction budget ahead of routine - // enrichment, and a spent budget only skips further RPCs: cached rows later in the order are still - // appended (stopping the loop at the cap dropped them from History). + // enrichment, then this cycle's discoveries, then the backlog. A spent budget only skips further + // RPCs: cached rows later in the order are still appended (stopping the loop at the cap dropped them + // from History), and what was skipped is handed back as unenrichedTxids. std::vector enrichOrder; - enrichOrder.reserve(knownTxids.size()); + enrichOrder.reserve(knownTxids.size() + backlog.size()); for (const auto& txid : knownTxids) { if (snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid); } for (const auto& txid : knownTxids) { if (!snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid); } + for (const auto& txid : backlog) { + if (!knownTxids.count(txid) && !snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid); + } int viewTxCount = 0; for (const std::string* txidPtr : enrichOrder) { @@ -1202,9 +1309,12 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr } } - if (!trackedSend && snapshot.fullyEnrichedTxids.count(txid)) continue; + if (!trackedSend && snapshot.fullyEnrichedTxids.count(txid) && !rowBacklog.count(txid)) continue; - if (viewTxCount >= maxViewTransactionsPerCycle) continue; + if (viewTxCount >= maxViewTransactionsPerCycle) { + if (!trackedSend) result.unenrichedTxids.insert(txid); + continue; + } ++viewTxCount; try { @@ -1216,11 +1326,22 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr appendTrackedSendPlaceholder(result.transactions, txid, hasRawTransaction ? &rawTransaction : nullptr); + } else { + result.unenrichedTxids.insert(txid); } continue; } auto entry = parseViewTransactionCacheEntry(viewTransaction); + const bool storedRow = rowBacklog.count(txid) > 0; + if (storedRow && entry.timestamp == 0) { + const auto settled = settledStoredMeta.find(txid); + if (settled != settledStoredMeta.end()) { + entry.timestamp = settled->second.first; + entry.confirmations = settled->second.second; + } + } + const std::size_t rowsBeforeView = result.transactions.size(); appendViewTransactionOutputs(result.transactions, txid, entry); // (Chat metadata is harvested only from RECEIVED notes, not our own outgoing memos — see // the cached-view branch above.) @@ -1246,6 +1367,21 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr } } + if (storedRow) { + // A stored send rebuilt without a date (gettransaction failed): keep its stored rows (still + // pinned) and retry next cycle, rather than cache an undated view for the session. + bool undated = false; + for (std::size_t i = rowsBeforeView; i < result.transactions.size(); ++i) { + if (result.transactions[i].txid == txid && result.transactions[i].timestamp == 0) undated = true; + } + if (undated) { + result.transactions.resize(rowsBeforeView); + result.unenrichedTxids.insert(txid); + continue; + } + pinnedRows.erase(txid); + } + if (trackedSend && !hasTransactionType(result.transactions, txid, "send")) { if (!hasRawTransaction) hasRawTransaction = tryFetchRawTransaction(rpc, txid, rawTransaction); appendTrackedSendPlaceholder(result.transactions, @@ -1275,13 +1411,20 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr } } } catch (const std::exception& e) { - (void)e; if (trackedSend) { json rawTransaction; bool hasRawTransaction = tryFetchRawTransaction(rpc, txid, rawTransaction); appendTrackedSendPlaceholder(result.transactions, txid, hasRawTransaction ? &rawTransaction : nullptr); + } else if (const auto* rpcError = dynamic_cast(&e); + rpcError && rpcError->code == -5) { + // "Invalid or non-wallet transaction id": definitive. Cache an empty view so it isn't + // asked again, and let a stored row for it go. + result.newViewTxEntries[txid] = TransactionViewCacheEntry{}; + pinnedRows.erase(txid); + } else { + result.unenrichedTxids.insert(txid); // anything else may be transient: retry } } } @@ -1289,17 +1432,21 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr appendExtractedHushChatMetadata(result.hushChatMetadata, hushChatReceivedOutputs); if (!snapshot.previousTransactions.empty()) { + std::unordered_set pinned = snapshot.pendingOpids; + pinned.insert(pinnedRows.begin(), pinnedRows.end()); if (transactionRpcError) { - appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, true, snapshot.pendingOpids); + appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, true, pinned); } else if (snapshot.shieldedAddresses.empty() || !result.shieldedScanComplete || result.shieldedAddressesScanned < result.shieldedAddressCount) { - appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, false, snapshot.pendingOpids); + appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, false, pinned); + } else if (!pinnedRows.empty()) { + // A full rebuild, except the stored rows of sends still waiting to be re-enriched. + appendPinnedPreviousTransactions(result.transactions, snapshot.previousTransactions, pinnedRows); } } sortTransactionsNewestFirst(result.transactions); - result.scanMs = std::chrono::duration( - std::chrono::steady_clock::now() - scanStart).count(); + result.scanMs = msSince(scanStart); return result; } @@ -1310,6 +1457,7 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe int pageSize) { TransactionRefreshResult result; + const auto scanStart = std::chrono::steady_clock::now(); result.blockHeight = currentBlockHeight; result.transactions = snapshot.previousTransactions; result.shieldedAddressCount = snapshot.shieldedAddresses.size(); @@ -1360,9 +1508,12 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe if (snapshot.maxShieldedReceiveScans > 0) { shieldedLimit = std::min(shieldedLimit, snapshot.maxShieldedReceiveScans); } + // Without the probe (the scan throttle is closed) the poll is listtransactions only. + if (snapshot.skipShieldedScans) shieldedLimit = 0; for (std::size_t index = shieldedStart; index < shieldedStart + shieldedLimit; ++index) { const auto& address = snapshot.shieldedAddresses[index]; + const auto callStart = std::chrono::steady_clock::now(); try { std::vector scannedTransactions; std::set scannedTxids; @@ -1384,18 +1535,27 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe } if (currentBlockHeight >= 0) result.shieldedScanHeights[address] = currentBlockHeight; ++result.shieldedAddressesScanned; + ++result.shieldedScansSucceeded; + result.receiveScanOkMs += std::chrono::duration( + std::chrono::steady_clock::now() - callStart).count(); } catch (const std::exception& e) { + ++result.shieldedAddressesScanned; DEBUG_LOGF("recent z_listreceivedbyaddress error: %s\n", e.what()); } } - result.nextShieldedScanStartIndex = - (shieldedStart + shieldedLimit >= snapshot.shieldedAddresses.size()) ? 0 : shieldedStart + shieldedLimit; + if (snapshot.skipShieldedScans) { + result.nextShieldedScanStartIndex = shieldedStart; + } else { + result.nextShieldedScanStartIndex = + (shieldedStart + shieldedLimit >= snapshot.shieldedAddresses.size()) ? 0 : shieldedStart + shieldedLimit; + } result.shieldedScanComplete = true; appendExtractedHushChatMetadata(result.hushChatMetadata, hushChatReceivedOutputs); sortTransactionsNewestFirst(result.transactions); + result.scanMs = std::chrono::duration(std::chrono::steady_clock::now() - scanStart).count(); return result; } diff --git a/src/services/network_refresh_service.h b/src/services/network_refresh_service.h index 2e5a885..68445cf 100644 --- a/src/services/network_refresh_service.h +++ b/src/services/network_refresh_service.h @@ -35,6 +35,13 @@ public: virtual ~RefreshRpcGateway() = default; virtual nlohmann::json call(const std::string& method, const nlohmann::json& params) = 0; + // A call allowed to outlast the default timeout (a whole-wallet sweep on a large wallet). + // Gateways without a separate timeout just call(). + virtual nlohmann::json callLong(const std::string& method, const nlohmann::json& params, long timeoutSec) + { + (void)timeoutSec; + return call(method, params); + } }; enum class Job { @@ -175,6 +182,9 @@ public: double amount = 0.0; // note value, DRGX bool locked = false; // tied up by an in-flight send int confirmations = 0; // TRUE depth (rawconfirmations when present, else confirmations) + std::string address; // the z-address holding it + std::string txid; // the transaction that created it + bool change = false; // change from the wallet's own spend (not a receive) }; struct AddressRefreshResult { @@ -221,14 +231,25 @@ public: std::set miningAddresses; std::unordered_map shieldedScanHeights; std::size_t shieldedScanStartIndex = 0; - std::size_t maxShieldedReceiveScans = 0; - // How many blocks the tip may advance past an address's last scan before it counts as stale - // and needs re-scanning. 0 = strict (must be scanned at the exact current tip). A small - // tolerance lets a multi-cycle pass over many shielded addresses COMPLETE even though new - // blocks arrive mid-pass — otherwise the "scanned at tip" bar moves every block and the scan - // (and the transactions_dirty_ flag it drives) never finishes. It also naturally throttles - // full rescans to roughly once per `tolerance` blocks. - int shieldedScanTipTolerance = 0; + std::size_t maxShieldedReceiveScans = 0; // receive scans this cycle; 0 = no limit + // The pass's floor (services::ShieldedHistoryPass): an address scanned below it is scanned again. + // -1: only never-scanned addresses (and priority ones) are. + int shieldedScanFloorHeight = -1; + // Scanned first, whatever the cursor, when never scanned or scanned below their floor: new + // addresses, the chat reply address, addresses holding notes the history doesn't show yet. + std::unordered_map priorityScanFloors; + bool skipShieldedScans = false; // a Light cycle (or a probe-less recent poll): no receive scans + bool skipSentSweep = false; + int sentScanFromHeight = -1; // the sweep's watermark; -1 = sweep the whole wallet + std::size_t transparentRowHint = 0; // listtransactions rows last time: sizes the first page + // Same tip and nothing changed since the rows on screen were fetched: reuse their transparent rows + // instead of paging listtransactions again. + bool reuseTransparentRows = false; + std::unordered_set enrichBacklogTxids; // discovered earlier, not enriched yet + // Run a windowed sent sweep even though the receive pass isn't complete (the caller rate-limits + // this), so untracked sends (from the Console, or another wallet on the same seed) don't wait for a + // whole pass. The whole-wallet sweep stays tied to a completed pass. + bool allowMidPassSentSweep = false; }; struct TransactionRefreshResult { @@ -245,9 +266,23 @@ public: std::size_t shieldedAddressCount = 0; std::unordered_map shieldedScanHeights; bool shieldedScanComplete = true; - // Wall-clock spent in the history scan (z_listreceivedbyaddress — O(mapWallet), holds cs_main). - // Lets the caller throttle the routine full rescan by its measured cost (txRefreshDue()). + // Wall-clock spent in the whole cycle: listtransactions, the receive scans and the sent sweep each + // hold cs_main. Lets the caller throttle wallet-wide scans by their measured cost (txRefreshDue()). double scanMs = 0.0; + std::size_t shieldedScansSucceeded = 0; // shieldedAddressesScanned counts attempts + double receiveScanOkMs = 0.0; // time in the receive scans that succeeded + std::size_t transparentRowCount = 0; // rows the listtransactions chain returned + bool transparentChainOk = false; + bool transparentRowsReused = false; + bool sentSweepRan = false; + bool sentSweepOk = false; + bool sentSweepFull = false; + int sentSweepHeight = -1; // the tip a successful sweep covered + double sentSweepMs = 0.0; // time in the sweep (part of scanMs) + rpc::RpcFailureKind sentSweepErrKind = rpc::RpcFailureKind::None; + // Txids found but not enriched (over the per-cycle cap, or the view failed transiently): the caller + // keeps them and feeds them back as enrichBacklogTxids. + std::unordered_set unenrichedTxids; }; struct OperationStatusPollResult { diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index e798832..d851958 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -27,6 +27,7 @@ #include "rpc/rpc_failure.h" #include "rpc/rpc_busy_gate.h" #include "resources/embedded_resources.h" +#include "services/history_scan_policy.h" #include "services/network_refresh_service.h" #include "services/refresh_scheduler.h" #include "services/note_consolidation.h" @@ -695,6 +696,15 @@ public: responses[method].push_back({nlohmann::json(), true, {}, std::move(exception)}); } + int longCalls = 0; + long lastLongTimeout = 0; + nlohmann::json callLong(const std::string& method, const nlohmann::json& params, long timeoutSec) override + { + ++longCalls; + lastLongTimeout = timeoutSec; + return call(method, params); // recorded like any call, so the pinned call sequences still hold + } + nlohmann::json call(const std::string& method, const nlohmann::json& params) override { calls.push_back({method, params}); @@ -2129,15 +2139,15 @@ void testNetworkRefreshRpcCollectors() EXPECT_EQ(pagedResult.transactions.size(), static_cast(1001)); EXPECT_EQ(pagedResult.transactions[0].txid, std::string("paged-1000")); - // shieldedScanTipTolerance: an address scanned a few blocks below the tip is re-scanned when - // strict (tolerance 0) but SKIPPED within tolerance — this is what lets a multi-block pass - // complete (and unsticks the perpetual "refreshing history" state). + // The pass floor: an address scanned below it is scanned again; without a floor (no pass running) + // only never-scanned addresses are. A pass's floor is its start height, so it completes however far + // the tip moves meanwhile (this is what unsticks the perpetual "refreshing history" state). { Refresh::TransactionRefreshSnapshot strictSnapshot; strictSnapshot.shieldedAddresses = {"zs-a", "zs-b"}; strictSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}}; strictSnapshot.maxShieldedReceiveScans = 4; - strictSnapshot.shieldedScanTipTolerance = 0; // strict: heights 100 are stale at tip 102 + strictSnapshot.shieldedScanFloorHeight = 102; // a pass that started at 102: both below it MockRefreshRpc strictRpc; strictRpc.addResponse("listtransactions", json::array()); strictRpc.addResponse("z_listreceivedbyaddress", json::array()); @@ -2149,8 +2159,7 @@ void testNetworkRefreshRpcCollectors() Refresh::TransactionRefreshSnapshot tolerantSnapshot; tolerantSnapshot.shieldedAddresses = {"zs-a", "zs-b"}; tolerantSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}}; - tolerantSnapshot.maxShieldedReceiveScans = 4; - tolerantSnapshot.shieldedScanTipTolerance = 5; // 100 >= 102-5 -> still fresh + tolerantSnapshot.maxShieldedReceiveScans = 4; // no floor: no pass is running MockRefreshRpc tolerantRpc; tolerantRpc.addResponse("listtransactions", json::array()); // No z_listreceivedbyaddress responses: the addresses must be skipped (else call() throws). @@ -2279,6 +2288,7 @@ void testNetworkRefreshRpcCollectors() staleProgressSnapshot.shieldedAddresses = {"zs-current", "zs-stale", "zs-missing"}; staleProgressSnapshot.shieldedScanHeights = {{"zs-current", 500}, {"zs-stale", 499}}; staleProgressSnapshot.maxShieldedReceiveScans = 1; + staleProgressSnapshot.shieldedScanFloorHeight = 500; staleProgressSnapshot.previousTransactions.push_back(dragonx::TransactionInfo{ "current-shielded", "receive", 0.75, 110, 12, "zs-current", "", "" }); @@ -9840,6 +9850,632 @@ void testScopeScanAndLockedNotes() EXPECT_FALSE(noteIsLocked(json::array())); } +// C43/C53/C52: the pure history-scan policy (services/history_scan_policy). +void testHistoryScanPolicy() +{ + using namespace dragonx::services; + using dragonx::rpc::RpcFailureKind; + // A routine re-pass is due after enough blocks that it costs ~10% of wall-clock, within [8, 500]. + EXPECT_EQ(historyStaleAfterBlocks(0, 0.0), 8); + EXPECT_EQ(historyStaleAfterBlocks(4, 500.0), 8); // 2 s of scans: the floor + EXPECT_EQ(historyStaleAfterBlocks(30, 10000.0), 84); // 300 s / (36 s * 10%) = 83.3 + EXPECT_EQ(historyStaleAfterBlocks(1000, 60000.0), 500); // capped + EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits::quiet_NaN()), 8); + EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits::infinity()), 500); + // The per-address cost estimate rises at once and decays 25% per sample; no sample, no change. + EXPECT_NEAR(updateScanCostEstimateMs(0.0, 400.0), 400.0, 1e-9); + EXPECT_NEAR(updateScanCostEstimateMs(400.0, 800.0), 800.0, 1e-9); + EXPECT_NEAR(updateScanCostEstimateMs(800.0, 400.0), 700.0, 1e-9); + EXPECT_NEAR(updateScanCostEstimateMs(800.0, 0.0), 800.0, 1e-9); + // listtransactions' first page: last time's rows plus headroom, within [1000, 5000]. + EXPECT_EQ(transparentPageSize(0), 1000); + EXPECT_EQ(transparentPageSize(900), 1100); + EXPECT_EQ(transparentPageSize(1500), 1700); + EXPECT_EQ(transparentPageSize(4800), 5000); + EXPECT_EQ(transparentPageSize(100000), 5000); + // The sent sweep: the whole wallet without a usable watermark, else a depth window. Never 3 params. + EXPECT_TRUE(sentSweepIsFull(0, 50)); + EXPECT_TRUE(sentSweepIsFull(100, -1)); + EXPECT_TRUE(sentSweepIsFull(100, 101)); // a watermark above the tip (reorg) + EXPECT_FALSE(sentSweepIsFull(100, 100)); + EXPECT_EQ(sentSweepParams("zs-one", 100, -1).dump(), std::string("[\"zs-one\",0]")); + const auto windowed = sentSweepParams("zs-one", 110, 100); + EXPECT_EQ(windowed.dump(), std::string("[\"zs-one\",0,2,110]")); + EXPECT_TRUE(windowed.size() == 4 && windowed[1].is_number_integer() && windowed[2].is_number_integer() && + windowed[3].is_number_integer()); + // Backoff after whole-wallet sweeps that failed after the daemon did the work. + EXPECT_EQ(fullSentSweepRetryDelaySec(0), 0L); + EXPECT_EQ(fullSentSweepRetryDelaySec(1), 600L); + EXPECT_EQ(fullSentSweepRetryDelaySec(2), 3600L); + EXPECT_EQ(fullSentSweepRetryDelaySec(3), 21600L); + EXPECT_EQ(fullSentSweepRetryDelaySec(9), 21600L); + EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::ReplyTimeout)); + EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::AppError)); + EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::Other)); + for (auto kind : {RpcFailureKind::Deferred, RpcFailureKind::Aborted, RpcFailureKind::NotListening, + RpcFailureKind::QueueFull, RpcFailureKind::Warmup}) { + EXPECT_FALSE(fullSentSweepFailureCounts(kind)); + } + // The between-blocks probe takes the address scanned longest ago, never-scanned first. + EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"b", 40}, {"c", 60}}), static_cast(1)); + EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"c", 60}}), static_cast(1)); + EXPECT_EQ(oldestScannedAddressIndex({}, {}), static_cast(0)); + + // decideTxCycle. + auto cycle = [](void (*setup)(TxCycleInputs&)) { + TxCycleInputs in; + in.heightKnown = true; + setup(in); + return decideTxCycle(in); + }; + EXPECT_TRUE(cycle([](TxCycleInputs&) {}) == TxCycle::None); + // A new block: a Full cycle only when the throttle allows; otherwise it waits. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.gateOpen = true; }) == TxCycle::Full); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; }) == TxCycle::None); + // Explicit scan work skips the throttle; routine work waits for it. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; }) == TxCycle::Full); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; }) == TxCycle::None); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; in.gateOpen = true; }) == TxCycle::Full); + // A mutation or the first load gets rows now through a Light cycle (no wallet-wide z-scans). + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.dirty = true; }) == TxCycle::Light); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.neverFetched = true; }) == TxCycle::Light); + // Tracking a send alone needs a new tip for a Light cycle, so it can't loop them at one tip. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; in.newTip = true; }) == TxCycle::Light); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; }) == TxCycle::None); + // Nothing scans while syncing or before the tip is known, explicit work included. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.dirty = true; in.syncing = true; }) == + TxCycle::Light); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.heightKnown = false; }) == TxCycle::None); + // The between-blocks poll probes one address only when the throttle allows. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; }) == TxCycle::Recent); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; }) == TxCycle::RecentWithProbe); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; in.syncing = true; }) == + TxCycle::Recent); + // At a new tip the cheap poll runs only while the throttle holds the Full cycle back. + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; }) == TxCycle::Recent); + EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; in.gateOpen = true; }) == + TxCycle::Full); +} + +namespace { +// Answers every call with an empty array and records the methods (for simulations). +class RecordingRefreshRpc : public dragonx::services::NetworkRefreshService::RefreshRpcGateway { +public: + std::vector methods; + nlohmann::json call(const std::string& method, const nlohmann::json&) override + { + methods.push_back(method); + return nlohmann::json::array(); + } +}; +} // namespace + +// C43 end to end: the policy, the pass and the real collector over 6 virtual hours on a large wallet (40 +// z-addresses at 10 s per wallet-wide call, 36 s blocks, the Overview cadence), mirroring the App glue in +// refreshTransactionData / startTransactionCycle. The first load completes, routine passes keep +// completing, and the scans hold cs_main for at most ~10% of the time once the first load is done. +void testHistoryDutyModel() +{ + using Refresh = dragonx::services::NetworkRefreshService; + using namespace dragonx::services; + constexpr int kAddresses = 40; + constexpr std::size_t kBudget = 4; + constexpr double kCallSec = 10.0; + constexpr double kTickSec = 10.0; + constexpr double kBlockSec = 36.0; + constexpr double kEndSec = 6 * 3600.0; + constexpr double kLateSec = 2 * 3600.0; + std::vector addrs; + for (int i = 0; i < kAddresses; ++i) addrs.push_back("zs-d" + std::to_string(i)); + std::unordered_map heights; + ShieldedHistoryPass pass; + bool initialComplete = false; + int lastTxTip = -1, watermark = -1; + std::size_t cursor = 0; + double lastScanAt = -1.0, lastScanCost = 0.0, busyUntil = 0.0, lateScanSec = 0.0; + int routinePassesDone = 0; + for (double now = 0.0; now < kEndSec; now += kTickSec) { + if (now < busyUntil) continue; // one cycle in flight at a time + const int tip = 1000 + static_cast(now / kBlockSec); + const int staleAfter = historyStaleAfterBlocks(addrs.size(), kCallSec * 1000.0); + const auto work = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete); + TxCycleInputs in; + in.neverFetched = lastTxTip < 0; + in.newTip = tip != lastTxTip; + in.explicitScanWork = work.explicitWork; + in.routineScanWork = work.routineWork; + in.gateOpen = lastScanAt < 0.0 || now - lastScanAt >= lastScanCost / kHistoryScanDuty; + in.heightKnown = true; + const TxCycle cycle = decideTxCycle(in); + if (cycle != TxCycle::Full && cycle != TxCycle::Light) continue; + const bool full = cycle == TxCycle::Full; + const bool routineAllowed = in.gateOpen; + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = addrs; + snap.shieldedScanHeights = heights; + if (full) { + const bool wasActive = pass.active(); + pass.begin(tip, addrs, heights, staleAfter, initialComplete, routineAllowed); + if (!wasActive && pass.active()) cursor = 0; + snap.shieldedScanFloorHeight = pass.scanFloor(tip, routineAllowed); + snap.shieldedScanStartIndex = cursor; + snap.maxShieldedReceiveScans = kBudget; + for (const auto& a : addrs) { + if (!heights.count(a)) snap.priorityScanFloors[a] = tip; + } + snap.sentScanFromHeight = watermark; + snap.skipSentSweep = (!routineAllowed && initialComplete) || (watermark >= 0 && watermark == tip); + snap.allowMidPassSentSweep = routineAllowed && watermark >= 0 && watermark < tip && + tip - watermark >= std::max(kHistoryMinStaleBlocks, staleAfter / 4); + } else { + snap.skipShieldedScans = true; + snap.skipSentSweep = true; + } + RecordingRefreshRpc rpc; + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 25); + double cost = 0.0, sweepCost = 0.0; + for (const auto& m : rpc.methods) { + if (m == "z_listreceivedbyaddress") cost += kCallSec; + if (m == "z_listsentbyaddress") { cost += kCallSec; sweepCost += kCallSec; } + } + busyUntil = now + cost; + if (now >= kLateSec) lateScanSec += cost; + lastTxTip = tip; + if (!full) continue; + lastScanCost = (result.sentSweepRan && result.sentSweepFull) ? std::max(0.001, cost - sweepCost) : cost; + lastScanAt = now + cost; + const bool wasActive = pass.active(); + const bool wasExplicit = pass.explicitPass(); + heights = result.shieldedScanHeights; + cursor = result.nextShieldedScanStartIndex; + if (result.sentSweepOk) watermark = result.sentSweepHeight; + pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded); + if (wasActive && !pass.active() && !wasExplicit) ++routinePassesDone; + const auto after = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete); + if (after.total > 0 && after.missing == 0 && !(pass.active() && pass.explicitPass())) initialComplete = true; + } + EXPECT_TRUE(initialComplete); + EXPECT_TRUE(routinePassesDone >= 3); + const double share = lateScanSec / (kEndSec - kLateSec); + EXPECT_TRUE(share <= 0.12); + EXPECT_TRUE(share >= 0.05); // and it does keep scanning (a stuck model would read 0) +} + +// C43: a pass re-scans every z-address once, ending when each is at or above the pass's start height. +void testShieldedHistoryPass() +{ + using namespace dragonx::services; + const std::vector addrs = {"z1", "z2", "z3"}; + ShieldedHistoryPass pass; + // First load: nothing scanned yet, all of it explicit work. + auto work = pass.evaluate(100, addrs, {}, 8, false); + EXPECT_EQ(work.total, static_cast(3)); + EXPECT_EQ(work.missing, static_cast(3)); + EXPECT_EQ(work.pending, static_cast(3)); + EXPECT_TRUE(work.explicitWork); + EXPECT_FALSE(work.routineWork); + // All scanned at 100: nothing is due until they're older than staleAfter. + std::unordered_map heights{{"z1", 100}, {"z2", 100}, {"z3", 100}}; + work = pass.evaluate(105, addrs, heights, 8, true); + EXPECT_EQ(work.pending, static_cast(0)); + EXPECT_FALSE(work.explicitWork); + EXPECT_FALSE(work.routineWork); + // Stale: a routine pass is due, and doesn't start without the throttle. + work = pass.evaluate(109, addrs, heights, 8, true); + EXPECT_EQ(work.pending, static_cast(3)); + EXPECT_FALSE(work.explicitWork); + EXPECT_TRUE(work.routineWork); + pass.begin(109, addrs, heights, 8, true, false); + EXPECT_FALSE(pass.active()); + EXPECT_EQ(pass.scanFloor(109, false), -1); + pass.begin(109, addrs, heights, 8, true, true); + EXPECT_TRUE(pass.active()); + EXPECT_FALSE(pass.explicitPass()); + EXPECT_EQ(pass.startHeight(), 109); + // A routine pass scans only in cycles the throttle allows. + EXPECT_EQ(pass.scanFloor(110, true), 109); + EXPECT_EQ(pass.scanFloor(110, false), -1); + // It ends once every address is at or above its floor, however far the tip has moved. + heights["z1"] = 112; + heights["z2"] = 115; + pass.finish(115, addrs, heights, 2, 2); + EXPECT_TRUE(pass.active()); // z3 is still below 109 + work = pass.evaluate(115, addrs, heights, 8, true); + EXPECT_EQ(work.pending, static_cast(1)); + EXPECT_TRUE(work.routineWork); + pass.finish(140, {}, heights, 4, 0); // a list blanked by a failed enumeration proves nothing + EXPECT_TRUE(pass.active()); + heights["z3"] = 140; + pass.finish(140, addrs, heights, 1, 1); + EXPECT_FALSE(pass.active()); + + // Before the first load completes, a pass is explicit: it may run back to back. + ShieldedHistoryPass first; + const std::unordered_map cached{{"z1", 10}, {"z2", 10}, {"z3", 10}}; // the history cache + first.begin(200, addrs, cached, 8, false, false); + EXPECT_TRUE(first.active()); + EXPECT_TRUE(first.explicitPass()); + EXPECT_EQ(first.scanFloor(200, false), 200); + EXPECT_TRUE(first.evaluate(201, addrs, cached, 8, false).explicitWork); + first.begin(150, addrs, cached, 8, false, false); // a reorg below the floor pulls it down + EXPECT_EQ(first.startHeight(), 150); + first.reset(); + EXPECT_FALSE(first.active()); + EXPECT_EQ(first.startHeight(), -1); + + // Scans that keep failing: explicit work turns routine (throttled) until one succeeds. + ShieldedHistoryPass stall; + stall.finish(100, addrs, {}, 2, 0); + EXPECT_TRUE(stall.stalled()); + work = stall.evaluate(100, addrs, {}, 8, false); + EXPECT_FALSE(work.explicitWork); + EXPECT_TRUE(work.routineWork); + stall.finish(100, addrs, {}, 1, 1); + EXPECT_FALSE(stall.stalled()); + EXPECT_TRUE(stall.evaluate(100, addrs, {}, 8, false).explicitWork); +} + +// C43 with the real collector: a pass over 10 addresses, 4 per cycle, finishes in 3 cycles although the tip +// moves 5 blocks per cycle (with the old "scanned within T of the tip" rule addresses went stale again +// mid-pass and a large wallet's pass never finished), and the sent sweep runs once, when it's done. +void testHistoryPassWithCollector() +{ + using Refresh = dragonx::services::NetworkRefreshService; + using namespace dragonx::services; + using json = nlohmann::json; + std::vector addrs; + std::unordered_map heights; + for (int i = 0; i < 10; ++i) { + addrs.push_back("zs-p" + std::to_string(i)); + heights[addrs.back()] = 100; + } + ShieldedHistoryPass pass; + int tip = 120; + pass.begin(tip, addrs, heights, 8, true, true); + EXPECT_TRUE(pass.active()); + std::size_t cursor = 0; + std::size_t scans = 0; + int cycles = 0, sweeps = 0, sweepCycle = -1; + while (pass.active() && cycles < 10) { + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = addrs; + snap.shieldedScanHeights = heights; + snap.maxShieldedReceiveScans = 4; + snap.shieldedScanFloorHeight = pass.scanFloor(tip, true); + snap.shieldedScanStartIndex = cursor; + snap.sentScanFromHeight = 100; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + for (int i = 0; i < 4; ++i) rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addResponse("z_listsentbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 0); + ++cycles; + scans += result.shieldedAddressesScanned; + for (const auto& call : rpc.calls) { + if (call.method == "z_listsentbyaddress") { ++sweeps; sweepCycle = cycles; } + } + heights = result.shieldedScanHeights; + cursor = result.nextShieldedScanStartIndex; + pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded); + tip += 5; + } + EXPECT_FALSE(pass.active()); + EXPECT_EQ(cycles, 3); + EXPECT_EQ(scans, static_cast(10)); + EXPECT_EQ(sweeps, 1); + EXPECT_EQ(sweepCycle, 3); + + // A new address while a routine pass is active and the throttle is closed: the cycle scans only it + // (priority, floor -1), not the routine pass's addresses, and the pass stays active. + std::vector many; + std::unordered_map manyHeights; + for (int i = 0; i < 12; ++i) { + many.push_back("zs-r" + std::to_string(i)); + if (i != 2) manyHeights[many.back()] = 90; + } + ShieldedHistoryPass routine; + routine.begin(100, many, manyHeights, 8, true, true); + EXPECT_TRUE(routine.active()); + Refresh::TransactionRefreshSnapshot explicitOnly; + explicitOnly.shieldedAddresses = many; + explicitOnly.shieldedScanHeights = manyHeights; + explicitOnly.maxShieldedReceiveScans = 4; + explicitOnly.shieldedScanFloorHeight = routine.scanFloor(105, false); + explicitOnly.shieldedScanStartIndex = 8; + explicitOnly.priorityScanFloors = {{many[2], 105}}; + explicitOnly.skipSentSweep = true; + EXPECT_EQ(explicitOnly.shieldedScanFloorHeight, -1); + MockRefreshRpc newAddrRpc; + newAddrRpc.addResponse("listtransactions", json::array()); + newAddrRpc.addResponse("z_listreceivedbyaddress", json::array()); + auto newAddr = Refresh::collectTransactionRefreshResult(newAddrRpc, explicitOnly, 105, 0); + EXPECT_TRUE(newAddrRpc.methodNames() == std::vector({"listtransactions", "z_listreceivedbyaddress"})); + EXPECT_EQ(newAddrRpc.calls[1].params, json::array({many[2], 0})); + EXPECT_EQ(newAddr.shieldedAddressesScanned, static_cast(1)); + routine.finish(105, many, newAddr.shieldedScanHeights, 1, 1); + EXPECT_TRUE(routine.active()); +} + +// C53/C52: the sent sweep, enrichment backlog and carry-forward, the adaptive page and row reuse. +void testHistoryCollectorSweepAndEnrichment() +{ + using Refresh = dragonx::services::NetworkRefreshService; + using json = nlohmann::json; + using dragonx::rpc::RpcFailureKind; + using dragonx::rpc::RpcTransportError; + using dragonx::rpc::RpcError; + auto freshSnapshot = []() { + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-one"}; + snap.shieldedScanHeights = {{"zs-one", 110}}; + return snap; + }; + auto countRows = [](const Refresh::TransactionRefreshResult& r, const std::string& txid) { + return static_cast(std::count_if(r.transactions.begin(), r.transactions.end(), + [&](const dragonx::TransactionInfo& t) { return t.txid == txid; })); + }; + + { // Past a watermark: a windowed sweep, an ordinary call. + auto snap = freshSnapshot(); + snap.sentScanFromHeight = 100; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listsentbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_TRUE(rpc.methodNames() == std::vector({"listtransactions", "z_listsentbyaddress"})); + EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]")); + EXPECT_EQ(rpc.longCalls, 0); + EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull); + EXPECT_EQ(result.sentSweepHeight, 110); + } + { // No watermark: the whole-wallet sweep, given the long timeout. + auto snap = freshSnapshot(); + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listsentbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0]")); + EXPECT_EQ(rpc.longCalls, 1); + EXPECT_EQ(rpc.lastLongTimeout, 120L); + EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk); + } + { // A failed sweep reports why (the caller's backoff), and keeps every previous row. + auto snap = freshSnapshot(); + snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""}); + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addException("z_listsentbyaddress", std::make_exception_ptr(RpcTransportError( + RpcFailureKind::ReplyTimeout, "RPC request failed: Timeout was reached", CURLE_OPERATION_TIMEDOUT, 0, true))); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_TRUE(result.sentSweepRan && !result.sentSweepOk); + EXPECT_TRUE(result.sentSweepErrKind == RpcFailureKind::ReplyTimeout); + EXPECT_EQ(countRows(result, "r-old"), 1); + } + { // skipSentSweep: no sweep at all. + auto snap = freshSnapshot(); + snap.skipSentSweep = true; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_TRUE(rpc.methodNames() == std::vector({"listtransactions"})); + EXPECT_FALSE(result.sentSweepRan); + } + { // A Light cycle: no receive scans, no sweep; the shielded rows on screen are carried. + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-a"}; + snap.skipShieldedScans = true; + snap.skipSentSweep = true; + snap.previousTransactions.push_back(dragonx::TransactionInfo{"s-old", "receive", 2.0, 60, 4, "zs-a", "", ""}); + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_TRUE(rpc.methodNames() == std::vector({"listtransactions"})); + EXPECT_EQ(countRows(result, "s-old"), 1); + } + { // Priority addresses are scanned before the cursor's. + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-a", "zs-b", "zs-c", "zs-d"}; + snap.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}, {"zs-c", 100}, {"zs-d", 100}}; + snap.shieldedScanFloorHeight = 105; + snap.priorityScanFloors = {{"zs-d", 105}}; + snap.maxShieldedReceiveScans = 2; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + Refresh::collectTransactionRefreshResult(rpc, snap, 105, 4); + EXPECT_EQ(rpc.calls[1].params, json::array({"zs-d", 0})); + EXPECT_EQ(rpc.calls[2].params, json::array({"zs-a", 0})); + } + { // Enrichment order: this cycle's discoveries, then the backlog; past the cap they're handed back. + auto snap = freshSnapshot(); + snap.skipSentSweep = true; + snap.enrichBacklogTxids = {"b1"}; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array({ + json{{"txid", "t1"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 1.0}, {"time", 70}, {"confirmations", 0}}, + json{{"txid", "t2"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 2.0}, {"time", 71}, {"confirmations", 0}} + })); + rpc.addResponse("z_viewtransaction", json{{"spends", json::array()}, {"outputs", json::array()}}); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 1); + EXPECT_EQ(rpc.calls[1].params, json::array({"t1"})); + EXPECT_EQ(result.unenrichedTxids.size(), static_cast(2)); + EXPECT_EQ(result.unenrichedTxids.count("t2"), static_cast(1)); + EXPECT_EQ(result.unenrichedTxids.count("b1"), static_cast(1)); + } + // A shielded send row from the history cache (an earlier session) with no cached view: re-enriched. + // The cycles below are full rebuilds (every address scanned, the pass complete), which drop rows that + // aren't rebuilt, except the "pinned" ones still waiting for their view. + auto rebuildSnapshot = [](std::vector previous) { + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-mine"}; + snap.shieldedScanHeights = {{"zs-mine", 100}}; + snap.shieldedScanFloorHeight = 110; + snap.skipSentSweep = true; + snap.previousTransactions = std::move(previous); + return snap; + }; + const dragonx::TransactionInfo oldSend{"zz-old", "send", -1.5, 900, 30, "zs-peer", "zs-mine", ""}; + { // The daemon answers: the row is rebuilt from the view, dated from the settled stored row (no + // gettransaction per row). + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addResponse("z_viewtransaction", json{ + {"spends", json::array({json{{"address", "zs-mine"}}})}, + {"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.5}}})}}); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4); + EXPECT_TRUE(rpc.methodNames() == std::vector({ + "listtransactions", "z_listreceivedbyaddress", "z_viewtransaction"})); + EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast(1)); + EXPECT_EQ(countRows(result, "zz-old"), 1); + for (const auto& t : result.transactions) { + if (t.txid != "zz-old") continue; + EXPECT_EQ(t.timestamp, static_cast(900)); + EXPECT_EQ(t.confirmations, 30); + } + } + { // A stored unsettled send still asks gettransaction; if that fails, the stored row stays (pinned), + // no undated view is cached, and it is retried. + const dragonx::TransactionInfo pendingSend{"zz-pending", "send", -1.0, 950, 0, "zs-peer", "zs-mine", ""}; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addResponse("z_viewtransaction", json{ + {"spends", json::array({json{{"address", "zs-mine"}}})}, + {"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.0}}})}}); + rpc.addFailure("gettransaction", "busy"); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({pendingSend}), 110, 4); + EXPECT_EQ(rpc.calls.back().method, std::string("gettransaction")); + EXPECT_EQ(result.newViewTxEntries.count("zz-pending"), static_cast(0)); + EXPECT_EQ(result.unenrichedTxids.count("zz-pending"), static_cast(1)); + EXPECT_EQ(countRows(result, "zz-pending"), 1); + for (const auto& t : result.transactions) { + if (t.txid == "zz-pending") EXPECT_EQ(t.timestamp, static_cast(950)); + } + } + { // Over the cap: still pinned, so the stored row survives the rebuild. + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 0); + EXPECT_EQ(countRows(result, "zz-old"), 1); + EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast(1)); + } + { // Only "not a wallet tx" (-5) is definitive: cached empty, and the row is let go. + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-5, "Invalid or non-wallet transaction id"))); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4); + EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast(1)); + EXPECT_TRUE(result.newViewTxEntries.at("zz-old").outgoing_outputs.empty()); + EXPECT_EQ(countRows(result, "zz-old"), 0); + } + for (int plainError = 0; plainError < 2; ++plainError) { // anything else: retried, the row kept + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + if (plainError) rpc.addFailure("z_viewtransaction", "boom"); + else rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-1, "something else"))); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4); + EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast(0)); + EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast(1)); + EXPECT_EQ(countRows(result, "zz-old"), 1); + } + { // A pinned send to two recipients keeps both rows (not just the first per txid), in a full rebuild... + const dragonx::TransactionInfo legA{"zz-two", "send", -1.0, 900, 30, "zs-peer-a", "zs-mine", ""}; + const dragonx::TransactionInfo legB{"zz-two", "send", -2.0, 900, 30, "zs-peer-b", "zs-mine", ""}; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({legA, legB}), 110, 0); + EXPECT_EQ(countRows(result, "zz-two"), 2); + // ...and in a cycle that carries rows (here a Light cycle). + Refresh::TransactionRefreshSnapshot light; + light.shieldedAddresses = {"zs-mine"}; + light.shieldedScanHeights = {{"zs-mine", 100}}; + light.skipShieldedScans = true; + light.skipSentSweep = true; + light.previousTransactions = {legA, legB}; + MockRefreshRpc lightRpc; + lightRpc.addResponse("listtransactions", json::array()); + auto carried = Refresh::collectTransactionRefreshResult(lightRpc, light, 110, 0); + EXPECT_EQ(countRows(carried, "zz-two"), 2); + EXPECT_EQ(carried.unenrichedTxids.count("zz-two"), static_cast(1)); + } + { // A windowed sweep mid-pass when allowed (rate-limited by the caller); never the whole-wallet one. + Refresh::TransactionRefreshSnapshot midPass; + midPass.shieldedAddresses = {"zs-a", "zs-b"}; + midPass.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}}; + midPass.shieldedScanFloorHeight = 110; + midPass.maxShieldedReceiveScans = 1; + midPass.sentScanFromHeight = 100; + midPass.allowMidPassSentSweep = true; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + rpc.addResponse("z_listreceivedbyaddress", json::array()); + rpc.addResponse("z_listsentbyaddress", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, midPass, 110, 4); + EXPECT_FALSE(result.shieldedScanComplete); + EXPECT_TRUE(result.sentSweepRan && !result.sentSweepFull); + EXPECT_EQ(rpc.calls.back().params.dump(), std::string("[\"zs-a\",0,2,110]")); + midPass.allowMidPassSentSweep = false; + MockRefreshRpc offRpc; + offRpc.addResponse("listtransactions", json::array()); + offRpc.addResponse("z_listreceivedbyaddress", json::array()); + EXPECT_FALSE(Refresh::collectTransactionRefreshResult(offRpc, midPass, 110, 4).sentSweepRan); + midPass.allowMidPassSentSweep = true; + midPass.sentScanFromHeight = -1; // no watermark: the whole-wallet sweep waits for the pass + MockRefreshRpc fullRpc; + fullRpc.addResponse("listtransactions", json::array()); + fullRpc.addResponse("z_listreceivedbyaddress", json::array()); + EXPECT_FALSE(Refresh::collectTransactionRefreshResult(fullRpc, midPass, 110, 4).sentSweepRan); + } + { // C52: the first page is sized from last time's rows. + auto snap = freshSnapshot(); + snap.skipSentSweep = true; + snap.transparentRowHint = 1500; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_EQ(rpc.calls[0].params, json::array({"", 1700, 0})); + EXPECT_TRUE(result.transparentChainOk); + } + { // Same tip, nothing changed: the transparent rows are reused, no listtransactions at all; a tracked + // send is rebuilt once (not reused and placeheld twice). + auto snap = freshSnapshot(); + snap.skipSentSweep = true; + snap.reuseTransparentRows = true; + snap.sendTxids = {"t-sent"}; + snap.previousTransactions = { + dragonx::TransactionInfo{"r1", "receive", 1.0, 80, 5, "R-one", "", ""}, + dragonx::TransactionInfo{"s1", "receive", 2.0, 81, 5, "zs-one", "", ""}, + dragonx::TransactionInfo{"t-sent", "send", -0.5, 82, 0, "R-two", "R-one", ""}, + }; + MockRefreshRpc rpc; + rpc.addResponse("z_viewtransaction", json()); + rpc.addResponse("gettransaction", json{{"time", 82}, {"confirmations", 0}}); + auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4); + EXPECT_EQ(rpc.calls[0].method, std::string("z_viewtransaction")); + EXPECT_TRUE(result.transparentRowsReused); + EXPECT_EQ(countRows(result, "r1"), 1); + EXPECT_EQ(countRows(result, "s1"), 1); + EXPECT_EQ(countRows(result, "t-sent"), 1); + } + { // The between-blocks poll without its probe: listtransactions only. + Refresh::TransactionRefreshSnapshot snap; + snap.shieldedAddresses = {"zs-a", "zs-b"}; + snap.maxShieldedReceiveScans = 1; + snap.skipShieldedScans = true; + MockRefreshRpc rpc; + rpc.addResponse("listtransactions", json::array()); + auto recent = Refresh::collectRecentTransactionRefreshResult(rpc, snap, 110); + EXPECT_TRUE(rpc.methodNames() == std::vector({"listtransactions"})); + EXPECT_EQ(recent.shieldedAddressesScanned, static_cast(0)); + } +} + int main() { testConnectionConfig(); @@ -9863,6 +10499,11 @@ int main() testParseMoneyZat(); testSpendableBalanceNeeded(); testOperationFilterParams(); + testHistoryScanPolicy(); + testShieldedHistoryPass(); + testHistoryPassWithCollector(); + testHistoryCollectorSweepAndEnrichment(); + testHistoryDutyModel(); testNetworkRefreshResultModels(); testTrackedSendRetirement(); testBalanceDisplayLerp();