DragonX/ObsidianDragon
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 10 Wiki Activity

perf(history): scan shielded history in bounded passes, not every block

Browse Source 
Every block re-ran z_listreceivedbyaddress for every z-address (O(wallet)
each, under cs_main) and a whole-wallet z_listsentbyaddress. On a 40-address
wallet that was ~80% of wall-clock. The history now advances in passes:

- an address is rescanned when it is stale by a number of blocks scaled to
  its measured cost (8 to 500), never-scanned and hinted addresses first
  (a hint: z_listunspent shows a note the history doesn't);
- z_listsentbyaddress covers only the blocks since the last good sweep
  (+100 reorg margin); the whole-wallet sweep runs once per connect, with
  a 10 min / 1 h / 6 h backoff after real failures;
- listtransactions pages are sized from last time, and an unchanged tip
  reuses the transparent rows;
- found-but-unenriched txids carry over as a backlog; only -5 is a
  definitive "not ours" for z_viewtransaction;
- between blocks, a cheap recent poll plus a one-address probe.

Duty model for a 40-address wallet at 10 s per scan: ~9% instead of ~80%.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DanS
2026-10-09 11:06:34 -05:00
co-authored by Claude Opus 5.5
parent 8b9a498eee
commit 879de37d18
10 changed files with 1645 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+3
View File
@@ -466,6 +466,7 @@ set(APP_SOURCES
src/app_security.cpp
src/app_sweep.cpp
src/app_wizard.cpp
src/services/history_scan_policy.cpp
src/services/network_refresh_service.cpp
src/services/note_consolidation.cpp
src/services/refresh_scheduler.cpp
@@ -613,6 +614,7 @@ endif()
set(APP_HEADERS
src/app.h
src/services/history_scan_policy.h
src/services/network_refresh_service.h
src/services/refresh_scheduler.h
src/services/wallet_security_controller.h
@@ -1148,6 +1150,7 @@ endif()
if(BUILD_TESTING)
add_executable(ObsidianDragonTests
tests/test_phase4.cpp
src/services/history_scan_policy.cpp
src/services/network_refresh_service.cpp
src/services/note_consolidation.cpp
src/services/refresh_scheduler.cpp
src/app.cpp
+30 -36
View File
@@ -1373,7 +1373,7 @@ void App::update()
removePendingSendTransactions(successOpids, false);
removePendingSendTransactions(failedOrStaleOpids, true);
// Transaction confirmed by daemon — force immediate data refresh
transactions_dirty_ = true;
markTransactionsDirty();
addresses_dirty_ = true;
last_tx_block_height_ = -1;
network_refresh_.markWalletMutationRefresh();
@@ -1417,21 +1417,11 @@ void App::update()
// Skip balance/tx/address refresh during warmup — RPC calls fail with -28 — and during a
// runtime rescan, when the daemon holds cs_main and these calls would block.
if (!state_.warming_up && !runtime_rescan_active_) {
if (network_refresh_.consumeDue(RefreshTimer::Transactions)) {
if (shouldRunWalletTransactionRefresh() && shouldRefreshTransactions()) {
// Throttle the routine new-block full history rescan (z_listreceivedbyaddress —
// O(mapWallet), holds cs_main) by its measured cost, so a large wallet doesn't
// re-scan every block and starve connection near the tip. Bypass when there's an
// explicit need: a dirty set, an in-progress multi-cycle shielded scan (which must
// continue to completion), or an in-flight send — all need fresh data immediately.
const bool txExplicitNeed = transactions_dirty_ || shielded_history_scan_pending_ ||
hasTransactionSendProgress() || !send_txids_.empty();
if (txExplicitNeed || txRefreshDue()) {
refreshTransactionData();
}
} else if (walletDataPage && shouldRefreshRecentTransactions()) {
refreshRecentTransactionData();
}
// History: refreshTransactionData() decides what this cycle may scan (services::decideTxCycle):
// wallet-wide scans only as their measured cost allows, except explicit work (the first load, an
// invalidation, a new address); sends and other mutations get their rows through Light cycles.
if (network_refresh_.consumeDue(RefreshTimer::Transactions) && shouldRunWalletTransactionRefresh()) {
refreshTransactionData();
}
// 0-conf chat: a DEDICATED ~2.5s mempool scan of the chat address, independent of the page's
// (slower, 10–15s) Transactions cadence — so incoming messages surface in a few seconds. The
@@ -5263,10 +5253,12 @@ void App::refreshNow()
{
// Trigger immediate refresh on all categories
network_refresh_.markImmediateRefresh();
transactions_dirty_ = true; // Force transaction list update
markTransactionsDirty(); // Force transaction list update
addresses_dirty_ = true; // Force address/balance update
last_tx_block_height_ = -1; // Reset tx cache
invalidateShieldedHistoryScanProgress(true);
// A full receive re-scan (explicit, so it runs back to back), but keep the sent sweep's watermark:
// every press would otherwise cost a whole-wallet z_listsentbyaddress.
invalidateShieldedHistoryScanProgress(true, /*keepSentScan=*/true);
}
void App::skipDaemonOutputBacklog()
@@ -5828,7 +5820,7 @@ void App::rescanBlockchain()
user_initiated_rescan_ = true; // user-triggered rescan → its completion should toast
state_.sync.rescan_progress = 0.0f;
state_.sync.rescan_status = decision.status;
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
invalidateShieldedHistoryScanProgress(true);
@@ -5882,7 +5874,7 @@ void App::repairWallet()
user_initiated_rescan_ = true; // user-triggered repair (implies rescan) → completion should toast
state_.sync.rescan_progress = 0.0f;
state_.sync.rescan_status = decision.status;
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
invalidateShieldedHistoryScanProgress(true);
@@ -7346,9 +7338,14 @@ std::string App::transactionSendProgressText() const
std::string App::transactionRefreshProgressText() const
{
using Job = services::NetworkRefreshService::Job;
if (!rpc_ || !rpc_->isConnected()) return {}; // Lite, or no node: there's no full-node history scan
bool running = network_refresh_.jobInProgress(Job::Transactions);
bool canRefresh = state_.connected && !state_.warming_up && !state_.isLocked();
if (!running && !(canRefresh && transactions_dirty_)) return {};
const auto work = historyScanWork();
// The first load spans several cycles, and the dirty flag now clears after each: keep the banner up
// between them while addresses are still to scan.
const bool initialLoad = !initial_history_scan_complete_ && work.total > 0;
if (!running && !(canRefresh && (transactions_dirty_ || (initialLoad && work.pending > 0)))) return {};
char buf[128];
@@ -7358,26 +7355,23 @@ std::string App::transactionRefreshProgressText() const
return buf;
}
// Initial bulk history load: the incremental shielded scan streams older transactions in over
// many refresh cycles, so the first batch can appear long before the history is complete. Keep
// the user informed (with a %) even after rows are on screen, so they know it's still loading.
// Progress = fraction of z-addresses scanned at least once (monotonic). Goes quiet for good once
// the first full scan completes; routine per-block re-scans don't re-trigger it.
if (!initial_history_scan_complete_ && !state_.z_addresses.empty()) {
std::size_t total = state_.z_addresses.size();
std::size_t scanned = std::min(shielded_history_scan_heights_.size(), total);
int pct = total > 0 ? static_cast<int>((scanned * 100) / total) : 0;
// Initial bulk history load: the shielded receive scans stream older transactions in over several
// refresh cycles, so the first batch can appear long before the history is complete. Keep the user
// informed (with a %) even after rows are on screen, so they know it's still loading. Progress is the
// share of z-addresses this pass has covered. Goes quiet for good once the first pass completes;
// routine re-scans afterwards don't re-trigger it.
if (initialLoad) {
const std::size_t total = work.total;
const std::size_t done = total - std::min(work.pending, total);
int pct = static_cast<int>((done * 100) / total);
if (pct > 99) pct = 99; // 100% is the "complete" state, which suppresses the banner
snprintf(buf, sizeof(buf), TR("tx_loading_history_progress"), pct);
return buf;
}
// Once history is on screen, stay quiet for routine background refreshes. The incremental
// shielded-receive scan re-dirties transactions_dirty_ on EVERY new block (each block moves the
// "scanned at tip" bar, invalidating prior per-address scans), so for a wallet with more
// z-addresses than the per-cycle scan budget the flag never clears — which left this banner lit
// indefinitely. New receives still appear live as they're scanned; only the INITIAL load
// (nothing displayed yet) and send enrichment (above) warrant a loading banner.
// Once history is on screen, stay quiet for routine background refreshes: new receives appear live
// as they're scanned; only the INITIAL load (nothing displayed yet) and send enrichment (above)
// warrant a loading banner.
if (!state_.transactions.empty()) return {};
if (!running && transactions_dirty_) return TR("tx_loading_queued");
src/app.h
+44 -7
View File
@@ -26,6 +26,7 @@
#include "daemon/stop_delivery.h"
#include "daemon/daemon_busy_detect.h"
#include "rpc/rpc_busy_gate.h"
#include "services/history_scan_policy.h"
#include "services/network_refresh_service.h"
#include "services/wallet_security_controller.h"
#include "services/wallet_security_workflow.h"
@@ -807,8 +808,13 @@ private:
void storeTransactionHistoryCacheIfAvailable();
void wipePendingTransactionHistoryCachePassphrase();
void resetTransactionHistoryCacheSession();
// The wallet locked: drop the cache key, keep the history and its scan progress (an unlock used to
// force a full re-scan of every address, every auto-lock cycle).
void lockTransactionHistoryCacheSession();
void pruneShieldedHistoryScanProgress();
void invalidateShieldedHistoryScanProgress(bool persistCache);
// Forget the receive-scan heights so every address is scanned again. keepSentScan leaves the sent
// sweep's watermark (and the enrichment backlog), so a plain refresh doesn't cost a whole-wallet sweep.
void invalidateShieldedHistoryScanProgress(bool persistCache, bool keepSentScan = false);
// Auto-balance pool selection: drive the periodic hashrate refresh and apply a
// freshly-completed snapshot (weighted-random pick + optional miner restart).
@@ -1361,7 +1367,10 @@ private:
// addressRefreshDue() / txRefreshDue(); only the routine periodic poll is throttled — explicit
// refreshes (tab switch, dirty set, in-flight send) call the refresh directly and bypass this.
double last_address_scan_ms_ = 0.0; // measured cost of the last address scan
double last_tx_scan_ms_ = 0.0; // measured cost of the last history scan
double last_tx_scan_ms_ = 0.0; // measured cost of the last wallet-wide history scan
// When that scan ran (wall-clock): the history throttle's clock. Not state_.last_tx_update, which
// every pending-send row edit and probe-less poll also stamps (that pushed the next scan out).
std::int64_t last_tx_scan_at_ = 0;
// Auto-clear for secrets copied to the clipboard. Only a hash of the copied secret is kept.
std::uint64_t clipboard_secret_hash_ = 0;
double clipboard_clear_deadline_ = 0.0;
@@ -1488,12 +1497,32 @@ private:
int last_tx_block_height_ = -1; // block height at last full tx fetch
static constexpr int MAX_VIEWTX_PER_CYCLE = 25; // cap z_viewtransaction calls per refresh
std::size_t shielded_history_scan_cursor_ = 0;
bool shielded_history_scan_pending_ = false;
// False until the first full shielded-history scan finishes. Drives the History tab's
// "Loading older history…" progress so the user knows transactions are still streaming in
// after the first batch appears; goes quiet for the routine per-block re-scans afterward.
bool initial_history_scan_complete_ = false;
std::unordered_map<std::string, int> shielded_history_scan_heights_;
// The receive-scan pass over every z-address (see services::ShieldedHistoryPass).
services::ShieldedHistoryPass history_pass_;
std::uint64_t history_scan_epoch_ = 0; // bumped when the scan heights are invalidated
std::uint64_t history_sent_epoch_ = 0; // bumped when the sent sweep's watermark is reset
int history_sent_scan_height_ = -1; // tip of the last successful z_listsentbyaddress; -1 = none
int history_full_sweep_failures_ = 0; // whole-wallet sweeps failed in a row (backoff)
std::time_t history_full_sweep_retry_at_ = 0;
std::unordered_set<std::string> history_enrich_backlog_; // txids found, not enriched yet
double history_addr_scan_ms_ = 0.0; // per-address z_listreceivedbyaddress cost (estimate)
std::size_t history_transparent_rows_ = 0; // rows the last listtransactions chain returned
int history_transparent_tip_ = -1; // ...and the tip it ran at (same tip: rows can be reused)
std::uint64_t tx_dirty_generation_ = 0; // bumped by markTransactionsDirty()
// Addresses holding notes the history doesn't show yet (from the address scan's z_listunspent): scanned
// first in the next wallet-wide cycle, so a receive or payout shows within one cycle on a large wallet
// instead of when the routine pass reaches its address. Each txid hints once per session after it is
// mined (a mempool hint doesn't use that up, so the row also catches its confirmation).
std::unordered_map<std::string, int> history_hint_floors_;
std::unordered_set<std::string> history_hinted_txids_;
// The z-addresses of the last successful address enumeration this session: a never-scanned address new
// to it (an imported key) resets the sent sweep's watermark, even if the list in between came back blank.
std::unordered_set<std::string> history_listed_z_;
// P4b: z_viewtransaction result cache — avoids re-calling the RPC for
// txids we've already enriched. Keyed by txid.
@@ -1733,8 +1762,17 @@ private:
void refreshData(); // Orchestrator: dispatches per-category refreshes
void refreshCoreData(bool allowBalance = true); // Balance + blockchain info (can use fast_worker_)
void refreshAddressData(); // Address lists + balances
void refreshTransactionData(); // Transaction list + z_viewtransaction enrichment
void refreshRecentTransactionData(); // Lightweight recent/unconfirmed tx poll
// Transaction history: decides the cycle (services::decideTxCycle) and starts it. Safe to call any
// time: it refuses when the node can't be scanned (not connected, locked, warming up, rescanning, busy).
void refreshTransactionData();
void startTransactionCycle(bool full, bool routineAllowed); // Full or Light cycle
void startRecentTransactionPoll(bool probe); // between-blocks poll
void markTransactionsDirty() { transactions_dirty_ = true; ++tx_dirty_generation_; }
std::vector<std::string> shieldedAddressList() const;
services::ShieldedHistoryPass::Work historyScanWork() const;
// From the address scan's z_listunspent: addresses holding received notes the history doesn't show
// yet (no row, or rows still unconfirmed while the note is mined) become priority scans.
void noteHistoryHintsFromUnspent(const std::vector<services::NetworkRefreshService::UnspentNoteLite>& notes);
bool refreshEncryptionState(); // Wallet encryption/lock state
void refreshBalance(); // Legacy: balance-only refresh (used by specific callers)
void refreshAddresses(); // Legacy: standalone address refresh
@@ -1744,13 +1782,12 @@ private:
bool effectivelySyncing() const; // syncing, or within the post-sync settle window (hysteresis)
bool balanceRefreshDue() const; // adaptive: enough time elapsed given the last balance-scan cost?
bool addressRefreshDue() const; // same adaptive back-off for the address scan (z_listunspent)
bool txRefreshDue() const; // same adaptive back-off for the history scan (z_listreceivedbyaddress)
bool txRefreshDue() const; // same adaptive back-off for wallet-wide history scans
// Shared duty-cycle rule: a scan may resume only once (lastScanMs / kScanDutyCycle) has elapsed since
// lastUpdate, so any single O(mapWallet) scan can occupy at most ~kScanDutyCycle of wall-clock.
bool scanRefreshDue(std::int64_t lastUpdate, double lastScanMs) const;
bool currentPageNeedsWalletDataRefresh() const;
bool shouldRunWalletTransactionRefresh() const;
bool shouldRefreshTransactions() const;
bool shouldRefreshRecentTransactions() const;
void checkAutoLock();
void checkIdleMining();
src/app_network.cpp
+328 -101
View File
@@ -99,8 +99,9 @@ std::string unencryptedTransactionHistoryCacheKey(const std::string& walletIdent
class AppRefreshRpcGateway final : public NetworkRefreshService::RefreshRpcGateway {
public:
AppRefreshRpcGateway(rpc::RPCClient& rpc, std::string source)
: rpc_(rpc), source_(std::move(source)) {}
// `maintenanceOwner`: the App whose rpc_ this is, for callLong() (see App::MaintenanceCallScope).
AppRefreshRpcGateway(rpc::RPCClient& rpc, std::string source, App* maintenanceOwner = nullptr)
: rpc_(rpc), source_(std::move(source)), owner_(maintenanceOwner) {}
json call(const std::string& method, const json& params) override
{
@@ -108,9 +109,20 @@ public:
return rpc_.call(method, params);
}
// Holds rpc_'s curl lock for up to timeoutSec: as a maintenance call, a main-thread disconnect
// aborts it rather than waiting for it (App::abortMaintenanceCalls()).
json callLong(const std::string& method, const json& params, long timeoutSec) override
{
std::optional<App::MaintenanceCallScope> maintenance;
if (owner_) maintenance.emplace(*owner_);
rpc::RPCClient::TraceScope trace(source_);
return rpc_.call(method, params, timeoutSec);
}
private:
rpc::RPCClient& rpc_;
std::string source_;
App* owner_ = nullptr;
};
const char* tracePageName(ui::NavPage page)
@@ -150,19 +162,6 @@ std::size_t shieldedReceiveScanBudget(ui::NavPage page)
return page == ui::NavPage::History ? 8u : 4u;
}
// How far the tip may drift past an address's last shielded scan before we re-scan it. A full pass
// scans ~budget addresses per refresh cycle (≈96 per block of wall time), so a wallet with many
// z-addresses takes several blocks to scan fully. With a strict (tolerance 0) "scanned at tip"
// check, new blocks arriving mid-pass would invalidate already-scanned addresses and the pass would
// never complete — leaving transactions_dirty_ (and its "refreshing history" banner + send-progress
// gate) stuck on forever. Scaling the tolerance with the address count lets the pass complete while
// keeping shielded-receive latency minimal for small wallets; it's capped for pathological sizes.
int shieldedScanTipTolerance(std::size_t shieldedAddressCount)
{
int t = 2 + static_cast<int>(shieldedAddressCount / 96);
return std::min(t, 50);
}
} // namespace
// ============================================================================
@@ -841,6 +840,9 @@ void App::onDisconnected(const std::string& reason)
opid_poll_in_progress_ = false;
opid_filter_unsupported_ = false; // a reconnect may be to a different daemon
last_total_balance_zat_.reset();
history_transparent_rows_ = 0; // the history's scan state (the pass, epochs and watermark are
history_transparent_tip_ = -1; // reset with the cache session below); cost estimates are kept
last_tx_scan_at_ = 0;
ui::ShieldDialog::invalidateScope();
address_validation_cache_dirty_ = true;
resetTransactionHistoryCacheSession();
@@ -1043,7 +1045,9 @@ bool App::addressRefreshDue() const
bool App::txRefreshDue() const
{
return scanRefreshDue(state_.last_tx_update, last_tx_scan_ms_);
// Keyed on the last wallet-wide scan, not state_.last_tx_update (pending-send row edits and probe-less
// polls stamp that too, which kept pushing the next scan out).
return scanRefreshDue(last_tx_scan_at_, last_tx_scan_ms_);
}
bool App::currentPageNeedsWalletDataRefresh() const
@@ -1059,7 +1063,7 @@ bool App::shouldRunWalletTransactionRefresh() const
{
if (currentPageNeedsWalletDataRefresh()) return true;
if (hasTransactionSendProgress() || !send_txids_.empty()) return true;
return transactions_dirty_ && !shielded_history_scan_pending_;
return transactions_dirty_;
}
void App::setCurrentPage(ui::NavPage page)
@@ -1083,7 +1087,7 @@ void App::setCurrentPage(ui::NavPage page)
network_refresh_.reset(RefreshTimer::Core);
break;
case NP::History:
transactions_dirty_ = true;
markTransactionsDirty();
refreshTransactionData();
network_refresh_.reset(RefreshTimer::Transactions);
break;
@@ -1108,25 +1112,14 @@ void App::setCurrentPage(ui::NavPage page)
// Screenshot sweep (tab-only + full-UI) + its demo data live in src/app_sweep.cpp.
bool App::shouldRefreshTransactions() const
{
// NOTE: this is block-height / dirty driven, NOT interval-gated. It returns true only when a new
// block arrived (currentBlocks != last_tx_block_height_), the history was never fetched, or
// something marked it dirty (tab entry, a send, a reorg, etc.). The Transactions timer only
// controls how often this CHECK runs; between blocks the lightweight recent-poll
// (shouldRefreshRecentTransactions / TxAge) handles mempool + unconfirmed deltas instead.
const int currentBlocks = state_.sync.blocks;
return network_refresh_.shouldRefreshTransactions(last_tx_block_height_,
currentBlocks,
transactions_dirty_);
}
bool App::shouldRefreshRecentTransactions() const
{
// No same-tip condition: at a new tip services::decideTxCycle runs a Full or Light cycle first when it
// can, so the cheap poll runs there only while the scan throttle holds the Full cycle back (it keeps
// transparent rows and confirmations live meanwhile, and doesn't consume the new tip).
using RefreshTimer = services::NetworkRefreshService::Timer;
return network_refresh_.isDue(RefreshTimer::TxAge)
&& last_tx_block_height_ >= 0
&& state_.sync.blocks == last_tx_block_height_
&& !state_.transactions.empty()
&& !transactions_dirty_
&& !addresses_dirty_;
@@ -1643,6 +1636,14 @@ void App::resetTransactionHistoryCacheSession()
wipePendingTransactionHistoryCachePassphrase();
transaction_history_cache_loaded_ = false;
invalidateShieldedHistoryScanProgress(false);
history_listed_z_.clear();
}
void App::lockTransactionHistoryCacheSession()
{
transaction_history_cache_.lockKey();
wipePendingTransactionHistoryCachePassphrase();
transaction_history_cache_loaded_ = false;
}
void App::pruneShieldedHistoryScanProgress()
@@ -1662,15 +1663,70 @@ void App::pruneShieldedHistoryScanProgress()
}
}
void App::invalidateShieldedHistoryScanProgress(bool persistCache)
void App::invalidateShieldedHistoryScanProgress(bool persistCache, bool keepSentScan)
{
shielded_history_scan_cursor_ = 0;
shielded_history_scan_pending_ = false;
initial_history_scan_complete_ = false; // a full re-scan is coming — show load progress again
shielded_history_scan_heights_.clear();
history_pass_.reset();
++history_scan_epoch_; // results of scans already in flight no longer apply
history_hint_floors_.clear();
history_hinted_txids_.clear();
if (!keepSentScan) {
history_sent_scan_height_ = -1; // the next sweep covers the whole wallet again
history_enrich_backlog_.clear();
history_full_sweep_failures_ = 0;
history_full_sweep_retry_at_ = 0;
++history_sent_epoch_;
}
if (persistCache) storeTransactionHistoryCacheIfAvailable();
}
std::vector<std::string> App::shieldedAddressList() const
{
// The same order as NetworkRefreshService::buildTransactionRefreshSnapshot.
std::vector<std::string> addresses;
addresses.reserve(state_.z_addresses.size());
for (const auto& address : state_.z_addresses) {
if (!address.address.empty()) addresses.push_back(address.address);
}
return addresses;
}
void App::noteHistoryHintsFromUnspent(
const std::vector<services::NetworkRefreshService::UnspentNoteLite>& notes)
{
const int tip = state_.sync.blocks;
if (tip <= 0 || notes.empty()) return;
std::unordered_map<std::string, bool> rowConfirmed; // txid -> any row of it is mined
for (const auto& transaction : state_.transactions) {
if (transaction.txid.empty()) continue;
bool& confirmed = rowConfirmed[transaction.txid];
confirmed = confirmed || transaction.confirmations >= 1;
}
for (const auto& note : notes) {
if (note.change || note.txid.empty() || note.address.empty()) continue;
if (history_hinted_txids_.count(note.txid)) continue; // once per txid: change/merges can't loop
const auto row = rowConfirmed.find(note.txid);
const bool noRow = row == rowConfirmed.end();
const bool staleUnconfirmed = !noRow && !row->second && note.confirmations >= 1;
if (!noRow && !staleUnconfirmed) continue;
int& floor = history_hint_floors_[note.address];
floor = std::max(floor, tip);
// A mempool hint doesn't use up the txid: once mined, its row (still at 0 conf) needs one more scan.
// No loop: an unmined note with a row matches neither case, and a mined one hints once.
if (note.confirmations >= 1) history_hinted_txids_.insert(note.txid);
}
}
services::ShieldedHistoryPass::Work App::historyScanWork() const
{
const auto addresses = shieldedAddressList();
return history_pass_.evaluate(state_.sync.blocks, addresses, shielded_history_scan_heights_,
services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_),
initial_history_scan_complete_);
}
bool App::ensureTransactionHistoryCacheUnlockedFor(const std::string& walletIdentity)
{
if (walletIdentity.empty()) return false;
@@ -1741,7 +1797,7 @@ void App::loadTransactionHistoryCacheIfAvailable()
}
confirmed_cache_block_ = loaded.tipHeight;
transaction_history_cache_loaded_ = true;
transactions_dirty_ = true;
markTransactionsDirty();
network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions);
}
@@ -1790,15 +1846,12 @@ void App::refreshData()
// as each completes, rather than waiting for the slowest phase.
refreshCoreData();
bool addressRefreshNeeded = addresses_dirty_;
bool walletDataPage = currentPageNeedsWalletDataRefresh();
if (addressRefreshNeeded)
// The address list first: the history needs it (the address callback marks the history due).
// refreshTransactionData() picks the cycle itself (Full / Light / the between-blocks poll / none).
if (addresses_dirty_)
refreshAddressData();
if (!addressRefreshNeeded && shouldRunWalletTransactionRefresh() && shouldRefreshTransactions())
else if (shouldRunWalletTransactionRefresh())
refreshTransactionData();
else if (!addressRefreshNeeded && walletDataPage && shouldRefreshRecentTransactions())
refreshRecentTransactionData();
// Always refresh peers here — refreshData() only runs on one-shot transitions (connect,
// warmup-complete, unlock), so this populates the status-bar peer count immediately on open
@@ -1857,7 +1910,7 @@ void App::refreshCoreData(bool allowBalance)
resetWitnessRescanProgress();
state_.sync.rescan_progress = 1.0f;
// Notes/witnesses were rebuilt — force a fresh history + balance pull.
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
invalidateShieldedHistoryScanProgress(true);
if (user_initiated_rescan_) {
@@ -2077,15 +2130,17 @@ void App::refreshAddressData()
if (!worker_ || !rpc_ || !state_.connected) return;
const std::size_t previousAddressCount = state_.z_addresses.size() + state_.t_addresses.size();
const std::string previousWalletIdentity = transactionHistoryCacheWalletIdentity();
const auto zList = shieldedAddressList();
std::unordered_set<std::string> previousZ(zList.begin(), zList.end());
auto addressSnapshot = address_validation_cache_dirty_
? NetworkRefreshService::AddressRefreshSnapshot{}
: NetworkRefreshService::buildAddressRefreshSnapshot(state_);
ui::NavPage tracePage = current_page_;
auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Addresses, *worker_, [this, previousAddressCount, previousWalletIdentity, addressSnapshot = std::move(addressSnapshot), tracePage]() -> rpc::RPCWorker::MainCb {
auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Addresses, *worker_, [this, previousAddressCount, previousWalletIdentity, previousZ = std::move(previousZ), addressSnapshot = std::move(addressSnapshot), tracePage]() -> rpc::RPCWorker::MainCb {
AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Address refresh"));
auto result = NetworkRefreshService::collectAddressRefreshResult(refreshRpc, addressSnapshot);
return [this, previousAddressCount, previousWalletIdentity, result = std::move(result)]() mutable {
return [this, previousAddressCount, previousWalletIdentity, previousZ, result = std::move(result)]() mutable {
const bool addrListOk = result.addressListOk; // capture before the move
const double scanMs = result.scanMs; // feed the adaptive throttle (addressRefreshDue)
if (scanMs > 0.0) last_address_scan_ms_ = scanMs;
@@ -2105,16 +2160,44 @@ void App::refreshAddressData()
const std::size_t currentAddressCount = state_.z_addresses.size() + state_.t_addresses.size();
const bool addressSetChanged = currentAddressCount != previousAddressCount ||
transactionHistoryCacheWalletIdentity() != previousWalletIdentity;
if (state_.transactions.empty() || addressSetChanged) {
if (addressSetChanged) {
invalidateShieldedHistoryScanProgress(false);
if (addrListOk) {
// Drop removed addresses' scan heights; the others stay valid (a changed set used to wipe
// every height, so one new address re-scanned the whole wallet). A z-address we never
// scanned that wasn't listed before (a key imported from the Console, say) may bring
// older sends with it, which a windowed sweep can't reach: sweep the whole wallet once.
pruneShieldedHistoryScanProgress();
// New: in neither the list at dispatch (GUI-created addresses are added to it before the
// refresh, and have no history) nor the last good listing (a blank list in between proves
// nothing), and never scanned.
bool newUnscannedZ = false;
std::unordered_set<std::string> listedZ;
for (const auto& address : state_.z_addresses) {
if (address.address.empty()) continue;
listedZ.insert(address.address);
if (!previousZ.count(address.address) && !history_listed_z_.count(address.address) &&
!shielded_history_scan_heights_.count(address.address))
newUnscannedZ = true;
}
transactions_dirty_ = true;
if (newUnscannedZ && (!previousZ.empty() || !history_listed_z_.empty())) {
history_sent_scan_height_ = -1;
history_full_sweep_failures_ = 0;
history_full_sweep_retry_at_ = 0;
++history_sent_epoch_;
}
history_listed_z_ = std::move(listedZ);
}
if (state_.transactions.empty() || addressSetChanged) {
markTransactionsDirty();
last_tx_block_height_ = -1;
network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions);
}
if (state_.transactions.empty()) loadTransactionHistoryCacheIfAvailable();
else storeTransactionHistoryCacheIfAvailable();
else if (addrListOk) storeTransactionHistoryCacheIfAvailable();
// Hints compare the notes with the history on screen: only once it's there (the cache loaded
// just above, or a pass is done). Before that every note looks missing, and the first pass scans
// every address anyway.
if (addrListOk && (!state_.transactions.empty() || initial_history_scan_complete_))
noteHistoryHintsFromUnspent(unspentNotes);
maybeFinishTransactionSendProgress();
// Addresses (hence identity + count) are now known — refresh the wallet index's cached
// metadata for the active wallet. Throttled: upsert() only writes on a real change.
@@ -2130,46 +2213,161 @@ void App::refreshAddressData()
void App::refreshTransactionData()
{
if (!worker_ || !rpc_ || !state_.connected) return;
if (addresses_dirty_) {
refreshAddressData();
network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions);
// The one entry point for history refreshes (the periodic timer, connect, tab entry, the address
// callback's re-route), so every caller gets the same rules. Wallet-wide scans hold the daemon's
// cs_main: never while it can't take them.
if (!worker_ || !rpc_ || !rpc_->isConnected() || !state_.connected) return;
if (state_.isLocked() || state_.warming_up || runtime_rescan_active_ || rpc_busy_gate_->busy()) return;
const int tip = state_.sync.blocks;
const auto work = historyScanWork();
services::TxCycleInputs in;
in.neverFetched = last_tx_block_height_ < 0;
in.newTip = tip != last_tx_block_height_;
in.dirty = transactions_dirty_;
in.sendTracking = !send_txids_.empty();
in.explicitScanWork = work.explicitWork;
in.routineScanWork = work.routineWork;
in.gateOpen = txRefreshDue();
in.syncing = effectivelySyncing();
in.heightKnown = tip > 0;
in.recentDue = currentPageNeedsWalletDataRefresh() && shouldRefreshRecentTransactions();
const auto cycle = services::decideTxCycle(in);
switch (cycle) {
case services::TxCycle::None:
return;
case services::TxCycle::Recent:
case services::TxCycle::RecentWithProbe:
startRecentTransactionPoll(cycle == services::TxCycle::RecentWithProbe);
return;
case services::TxCycle::Full:
case services::TxCycle::Light:
if (addresses_dirty_) {
// The history needs the current address list; the address callback marks it due again.
refreshAddressData();
network_refresh_.markDue(services::NetworkRefreshService::Timer::Transactions);
return;
}
startTransactionCycle(cycle == services::TxCycle::Full, in.gateOpen);
return;
}
}
void App::startTransactionCycle(bool full, bool routineAllowed)
{
const int currentBlocks = state_.sync.blocks;
if (last_tx_block_height_ < 0 || currentBlocks != last_tx_block_height_ ||
!shielded_history_scan_pending_) {
shielded_history_scan_cursor_ = 0;
shielded_history_scan_pending_ = false;
}
auto transactionSnapshot = NetworkRefreshService::buildTransactionRefreshSnapshot(
state_, viewtx_cache_, send_txids_);
transactionSnapshot.pendingOpids.insert(pending_opids_.begin(), pending_opids_.end());
if (settings_) transactionSnapshot.miningAddresses = settings_->getMiningAddresses();
transactionSnapshot.shieldedScanHeights = shielded_history_scan_heights_;
transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_;
transactionSnapshot.maxShieldedReceiveScans = shieldedReceiveScanBudget(current_page_);
transactionSnapshot.shieldedScanTipTolerance =
shieldedScanTipTolerance(transactionSnapshot.shieldedAddresses.size());
// When fully synced, widen the re-scan tolerance so the O(mapWallet) per-address history scan runs
// every several blocks instead of ~every 2 — the heaviest residual synced-state cs_main cost. The
// balance poll still surfaces incoming funds on its own cadence; only the detailed history list lags a
// few minutes. (During catch-up the tx refresh is suppressed by kSyncProfile, so this is synced-only.)
if (!effectivelySyncing())
transactionSnapshot.shieldedScanTipTolerance =
std::max(transactionSnapshot.shieldedScanTipTolerance, 8);
transactionSnapshot.transparentRowHint = history_transparent_rows_;
transactionSnapshot.reuseTransparentRows = !transactions_dirty_ && currentBlocks > 0 &&
history_transparent_tip_ == currentBlocks && !state_.transactions.empty();
transactionSnapshot.enrichBacklogTxids = history_enrich_backlog_;
if (full) {
const auto& addresses = transactionSnapshot.shieldedAddresses;
const bool wasActive = history_pass_.active();
history_pass_.begin(currentBlocks, addresses, shielded_history_scan_heights_,
services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_),
initial_history_scan_complete_, routineAllowed);
if (!wasActive && history_pass_.active()) shielded_history_scan_cursor_ = 0; // a new pass: from the top
transactionSnapshot.shieldedScanFloorHeight = history_pass_.scanFloor(currentBlocks, routineAllowed);
transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_;
transactionSnapshot.maxShieldedReceiveScans = shieldedReceiveScanBudget(current_page_);
auto listed = [&](const std::string& address) {
return std::find(addresses.begin(), addresses.end(), address) != addresses.end();
};
// Never-scanned addresses first, whatever the cursor (the explicit work: a new or imported
// address costs one scan, not a pass).
for (const auto& address : addresses) {
if (!shielded_history_scan_heights_.count(address)) transactionSnapshot.priorityScanFloors[address] = currentBlocks;
}
if (routineAllowed) {
// Addresses holding notes the history doesn't show yet (noteHistoryHintsFromUnspent).
for (const auto& [address, floor] : history_hint_floors_) {
if (listed(address)) transactionSnapshot.priorityScanFloors.emplace(address, floor);
}
// The chat reply address once per new tip, so messages also surface off the Chat tab.
if (!lite_wallet_ && chat::hushChatFeatureEnabledAtBuild() && chat_service_.hasIdentity() && settings_) {
const std::string chatZ = settings_->getChatReplyZaddr();
if (!chatZ.empty() && listed(chatZ)) transactionSnapshot.priorityScanFloors[chatZ] = currentBlocks;
}
}
transactionSnapshot.sentScanFromHeight = history_sent_scan_height_;
// A windowed sweep mid-pass too, now and then, so an untracked send (the Console, another wallet on
// the same seed) doesn't wait for a whole pass on a large wallet: at most every quarter of the pass
// cadence, and only in cycles the throttle allows.
const int staleAfter = services::historyStaleAfterBlocks(addresses.size(), history_addr_scan_ms_);
transactionSnapshot.allowMidPassSentSweep =
routineAllowed && history_sent_scan_height_ >= 0 && history_sent_scan_height_ < currentBlocks &&
currentBlocks - history_sent_scan_height_ >= std::max(services::kHistoryMinStaleBlocks, staleAfter / 4);
// The sweep runs once per tip, when the throttle allows (or the first load is still running),
// and backs off after whole-wallet sweeps that failed.
transactionSnapshot.skipSentSweep =
(!routineAllowed && initial_history_scan_complete_) ||
(history_sent_scan_height_ >= 0 && history_sent_scan_height_ == currentBlocks) ||
(history_sent_scan_height_ < 0 && std::time(nullptr) < history_full_sweep_retry_at_);
} else {
// Light: rows now (a send, a mutation), no wallet-wide z-scans.
transactionSnapshot.skipShieldedScans = true;
transactionSnapshot.skipSentSweep = true;
transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_;
}
const std::uint64_t scanEpoch = history_scan_epoch_;
const std::uint64_t sentEpoch = history_sent_epoch_;
const std::uint64_t dirtyGen = tx_dirty_generation_;
const bool snapshotHadZ = !transactionSnapshot.shieldedAddresses.empty();
ui::NavPage tracePage = current_page_;
auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks,
transactionSnapshot = std::move(transactionSnapshot), tracePage]() -> rpc::RPCWorker::MainCb {
AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Transaction refresh"));
auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks, full,
transactionSnapshot = std::move(transactionSnapshot), tracePage, scanEpoch, sentEpoch, dirtyGen,
snapshotHadZ]() -> rpc::RPCWorker::MainCb {
AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, full ? "Transaction refresh" : "Transaction refresh (light)"),
this);
auto result = NetworkRefreshService::collectTransactionRefreshResult(
refreshRpc, transactionSnapshot, currentBlocks, MAX_VIEWTX_PER_CYCLE);
return [this, result = std::move(result)]() mutable {
bool shieldedScanComplete = result.shieldedScanComplete;
if (result.scanMs > 0.0) last_tx_scan_ms_ = result.scanMs; // feed the adaptive throttle (txRefreshDue)
return [this, full, scanEpoch, sentEpoch, dirtyGen, snapshotHadZ, result = std::move(result)]() mutable {
const std::time_t now = std::time(nullptr);
const int resultTip = result.blockHeight;
const std::size_t attempted = result.shieldedAddressesScanned;
const std::size_t succeeded = result.shieldedScansSucceeded;
const bool sameScanEpoch = scanEpoch == history_scan_epoch_;
// Only history_sent_epoch_ invalidates the sent side (watermark, backoff, backlog): a plain
// refresh keeps the sent scan, so it doesn't throw away an in-flight whole-wallet sweep.
const bool sameSentEpoch = sentEpoch == history_sent_epoch_;
// Feed the throttle (txRefreshDue): Full cycles are the wallet-wide scans it meters. Not the
// whole-wallet sweep, which has its own cadence (once per connect or full invalidation, with a
// backoff): charging up to 120 s of it held routine scans back for 10x that after every connect.
if (full) {
double charged = result.scanMs;
if (result.sentSweepRan && result.sentSweepFull) charged = std::max(1.0, result.scanMs - result.sentSweepMs);
if (charged > 0.0) last_tx_scan_ms_ = charged;
last_tx_scan_at_ = now;
}
if (sameScanEpoch) {
if (succeeded > 0) {
history_addr_scan_ms_ = services::updateScanCostEstimateMs(
history_addr_scan_ms_, result.receiveScanOkMs / static_cast<double>(succeeded));
}
if (result.transparentChainOk) {
history_transparent_rows_ = result.transparentRowCount;
history_transparent_tip_ = resultTip;
}
}
if (sameSentEpoch) history_enrich_backlog_ = std::move(result.unenrichedTxids);
if (sameSentEpoch) {
if (result.sentSweepOk && result.sentSweepHeight > 0) {
history_sent_scan_height_ = result.sentSweepHeight;
history_full_sweep_failures_ = 0;
history_full_sweep_retry_at_ = 0;
} else if (result.sentSweepRan && result.sentSweepFull &&
services::fullSentSweepFailureCounts(result.sentSweepErrKind)) {
++history_full_sweep_failures_;
history_full_sweep_retry_at_ = now + services::fullSentSweepRetryDelaySec(history_full_sweep_failures_);
}
}
std::size_t nextShieldedScanStartIndex = result.nextShieldedScanStartIndex;
auto shieldedScanHeights = std::move(result.shieldedScanHeights);
NetworkRefreshService::TransactionCacheUpdate cacheUpdate{
@@ -2207,12 +2405,32 @@ void App::refreshTransactionData()
NetworkRefreshService::applyTransactionRefreshResult(
state_, cacheUpdate, std::move(result), std::time(nullptr));
pruneTrackedSendTxids();
shielded_history_scan_heights_ = std::move(shieldedScanHeights);
// Pass bookkeeping on the main thread, from the merged heights. Not from a cycle whose address
// list came back blank (enumeration failed): it scanned nothing and knows nothing.
if (full && sameScanEpoch && (snapshotHadZ || shielded_history_scan_heights_.empty())) {
shielded_history_scan_heights_ = std::move(shieldedScanHeights);
shielded_history_scan_cursor_ = nextShieldedScanStartIndex;
const auto addresses = shieldedAddressList();
// A list blanked by a failed enumeration after dispatch proves nothing: don't end the pass
// or call the first load complete from it.
if (!(snapshotHadZ && addresses.empty())) {
history_pass_.finish(resultTip, addresses, shielded_history_scan_heights_, attempted, succeeded);
for (auto it = history_hint_floors_.begin(); it != history_hint_floors_.end();) {
const auto height = shielded_history_scan_heights_.find(it->first);
if (height != shielded_history_scan_heights_.end() && height->second >= it->second)
it = history_hint_floors_.erase(it);
else
++it;
}
const auto work = historyScanWork();
if (work.total > 0 && work.missing == 0 &&
!(history_pass_.active() && history_pass_.explicitPass()))
initial_history_scan_complete_ = true;
}
}
storeTransactionHistoryCacheIfAvailable();
shielded_history_scan_cursor_ = nextShieldedScanStartIndex;
shielded_history_scan_pending_ = !shieldedScanComplete;
if (shieldedScanComplete) initial_history_scan_complete_ = true;
transactions_dirty_ = !shieldedScanComplete;
// A mutation that landed while this cycle ran isn't covered by it: stay dirty for it.
if (tx_dirty_generation_ == dirtyGen) transactions_dirty_ = false;
maybeFinishTransactionSendProgress();
};
}, 3);
@@ -2245,34 +2463,44 @@ void App::pruneTrackedSendTxids()
}
}
void App::refreshRecentTransactionData()
void App::startRecentTransactionPoll(bool probe)
{
if (!worker_ || !rpc_ || !state_.connected) return;
if (!shouldRefreshRecentTransactions()) return;
const int currentBlocks = state_.sync.blocks;
auto transactionSnapshot = NetworkRefreshService::buildTransactionRefreshSnapshot(
state_, viewtx_cache_, send_txids_);
transactionSnapshot.pendingOpids.insert(pending_opids_.begin(), pending_opids_.end());
if (settings_) transactionSnapshot.miningAddresses = settings_->getMiningAddresses();
transactionSnapshot.shieldedScanHeights = shielded_history_scan_heights_;
transactionSnapshot.shieldedScanStartIndex = shielded_history_scan_cursor_;
// Between blocks: recent listtransactions rows, plus (when the scan throttle allows) one receive scan
// of the address scanned longest ago. It no longer advances the pass cursor.
transactionSnapshot.skipShieldedScans = !probe;
transactionSnapshot.maxShieldedReceiveScans = 1;
transactionSnapshot.shieldedScanTipTolerance =
shieldedScanTipTolerance(transactionSnapshot.shieldedAddresses.size());
if (!effectivelySyncing()) // synced: widen the re-scan tolerance (see refreshTransactionData)
transactionSnapshot.shieldedScanTipTolerance =
std::max(transactionSnapshot.shieldedScanTipTolerance, 8);
transactionSnapshot.shieldedScanStartIndex = probe
? services::oldestScannedAddressIndex(transactionSnapshot.shieldedAddresses, shielded_history_scan_heights_)
: 0;
const std::uint64_t scanEpoch = history_scan_epoch_;
ui::NavPage tracePage = current_page_;
auto enqueued = network_refresh_.enqueue(services::NetworkRefreshService::Job::Transactions, *worker_, [this, currentBlocks,
transactionSnapshot = std::move(transactionSnapshot), tracePage]() -> rpc::RPCWorker::MainCb {
transactionSnapshot = std::move(transactionSnapshot), tracePage, probe, scanEpoch]() -> rpc::RPCWorker::MainCb {
AppRefreshRpcGateway refreshRpc(*rpc_, traceSource(tracePage, "Recent transaction poll"));
auto result = NetworkRefreshService::collectRecentTransactionRefreshResult(
refreshRpc, transactionSnapshot, currentBlocks);
return [this, result = std::move(result)]() mutable {
std::size_t nextShieldedScanStartIndex = result.nextShieldedScanStartIndex;
return [this, probe, scanEpoch, result = std::move(result)]() mutable {
// The poll doesn't consume the tip: newTip stays set until a Full or Light cycle applies at it,
// so the Full cycle still runs once the throttle allows. (Read here, not at dispatch, so a reset
// to -1 meanwhile is kept.)
int pollTip = last_tx_block_height_;
if (probe && result.shieldedAddressesScanned > 0) {
// The probe is a wallet-wide scan: it counts for the throttle.
if (result.scanMs > 0.0) last_tx_scan_ms_ = result.scanMs;
last_tx_scan_at_ = std::time(nullptr);
if (scanEpoch == history_scan_epoch_ && result.shieldedScansSucceeded > 0) {
history_addr_scan_ms_ = services::updateScanCostEstimateMs(
history_addr_scan_ms_, result.receiveScanOkMs / static_cast<double>(result.shieldedScansSucceeded));
}
}
auto shieldedScanHeights = std::move(result.shieldedScanHeights);
NetworkRefreshService::TransactionCacheUpdate cacheUpdate{
viewtx_cache_,
@@ -2280,7 +2508,7 @@ void App::refreshRecentTransactionData()
confirmed_tx_cache_,
confirmed_tx_ids_,
confirmed_cache_block_,
last_tx_block_height_
pollTip
};
// HushChat: decrypt & thread any harvested chat memos BEFORE `result` is moved (see the
// full-refresh path above for rationale; the store dedups across both paths).
@@ -2307,8 +2535,7 @@ void App::refreshRecentTransactionData()
}
NetworkRefreshService::applyTransactionRefreshResult(
state_, cacheUpdate, std::move(result), std::time(nullptr));
shielded_history_scan_heights_ = std::move(shieldedScanHeights);
shielded_history_scan_cursor_ = nextShieldedScanStartIndex;
if (scanEpoch == history_scan_epoch_) shielded_history_scan_heights_ = std::move(shieldedScanHeights);
storeTransactionHistoryCacheIfAvailable();
maybeFinishTransactionSendProgress();
};
@@ -2341,7 +2568,7 @@ bool App::refreshEncryptionState()
return [this, result]() {
NetworkRefreshService::applyWalletEncryptionResult(state_, result);
if (state_.isLocked()) {
resetTransactionHistoryCacheSession();
lockTransactionHistoryCacheSession();
} else if (state_.transactions.empty()) {
loadTransactionHistoryCacheIfAvailable();
} else {
@@ -5855,7 +6082,7 @@ void App::submitZSendMany(const std::string& from, const std::string& to, double
// A send changes address balances — refresh on next cycle
addresses_dirty_ = true;
// Force transaction list refresh so the sent tx appears immediately
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
requestForcedBalanceRefresh(); // the user's own send must update the balance now, past the throttle
network_refresh_.markWalletMutationRefresh();
@@ -6030,7 +6257,7 @@ void App::runtimeRescan(int startHeight)
user_initiated_rescan_ = true; // user clicked Rescan → completion should toast
state_.sync.rescan_progress = 0.0f;
state_.sync.rescan_status = "Rescanning from block " + std::to_string(startHeight) + "...";
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
invalidateShieldedHistoryScanProgress(true);
@@ -6052,7 +6279,7 @@ void App::runtimeRescan(int startHeight)
resetWitnessRescanProgress();
if (ok) {
state_.sync.rescan_progress = 1.0f;
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
invalidateShieldedHistoryScanProgress(true);
ui::Notifications::instance().success("Blockchain rescan complete");
src/app_security.cpp
+3 -3
View File
@@ -547,7 +547,7 @@ void App::lockWallet() {
if (ok) {
state_.locked = true;
state_.unlocked_until = 0;
resetTransactionHistoryCacheSession();
lockTransactionHistoryCacheSession(); // keeps the history's scan progress
lock_failure_warned_ = false;
DEBUG_LOGF("[App] Wallet locked\n");
} else {
@@ -642,7 +642,7 @@ void App::refreshWalletEncryptionState() {
settings_->save();
}
if (state_.locked) {
resetTransactionHistoryCacheSession();
lockTransactionHistoryCacheSession();
} else if (state_.transactions.empty()) {
loadTransactionHistoryCacheIfAvailable();
} else {
@@ -1799,7 +1799,7 @@ void App::renderDecryptWalletDialog() {
// Force address + peer refresh
invalidateAddressValidationCache();
addresses_dirty_ = true;
transactions_dirty_ = true;
markTransactionsDirty();
last_tx_block_height_ = -1;
refreshWalletEncryptionState();
src/services/history_scan_policy.cpp
+180
View File
@@ -0,0 +1,180 @@
// DragonX Wallet - ImGui Edition
// Copyright 2024-2026 The Hush Developers
// Released under the GPLv3
#include "history_scan_policy.h"
#include <algorithm>
#include <cmath>
namespace dragonx {
namespace services {
int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs)
{
if (addressCount == 0 || !(perAddressScanMs > 0.0)) return kHistoryMinStaleBlocks;
const double blocks = std::ceil((static_cast<double>(addressCount) * perAddressScanMs / 1000.0) /
(kHistoryBlockSeconds * kHistoryScanDuty));
if (!(blocks < kHistoryMaxStaleBlocks)) return kHistoryMaxStaleBlocks; // also NaN / inf
return std::max(kHistoryMinStaleBlocks, static_cast<int>(blocks));
}
double updateScanCostEstimateMs(double previousMs, double sampleMs)
{
if (!(sampleMs > 0.0)) return previousMs;
if (!(previousMs > 0.0) || sampleMs > previousMs) return sampleMs;
return 0.75 * previousMs + 0.25 * sampleMs;
}
int transparentPageSize(std::size_t previousRowCount)
{
const std::size_t want = previousRowCount + std::max<std::size_t>(200, previousRowCount / 10);
return static_cast<int>(std::clamp<std::size_t>(want, kTransparentPageMin, kTransparentPageMax));
}
bool sentSweepIsFull(int tip, int fromHeight)
{
return tip <= 0 || fromHeight < 0 || fromHeight > tip;
}
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight)
{
nlohmann::json params = nlohmann::json::array();
params.push_back(anyZAddress);
params.push_back(0);
if (sentSweepIsFull(tip, fromHeight)) return params;
const int maxDepth = (tip - fromHeight) + kSentSweepReorgMarginBlocks;
params.push_back(2); // filter type 2: depth
params.push_back(maxDepth); // at most this deep (mempool txs, depth 0, always pass)
return params;
}
long fullSentSweepRetryDelaySec(int consecutiveFailures)
{
if (consecutiveFailures <= 0) return 0;
if (consecutiveFailures == 1) return 600;
if (consecutiveFailures == 2) return 3600;
return 21600;
}
bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind)
{
using K = rpc::RpcFailureKind;
return kind == K::ReplyTimeout || kind == K::AppError || kind == K::Other;
}
std::size_t oldestScannedAddressIndex(const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights)
{
std::size_t best = 0;
int bestHeight = 0;
bool haveBest = false;
for (std::size_t i = 0; i < addresses.size(); ++i) {
const auto it = heights.find(addresses[i]);
if (it == heights.end()) return i; // never scanned: first
if (!haveBest || it->second < bestHeight) {
best = i;
bestHeight = it->second;
haveBest = true;
}
}
return best;
}
int ShieldedHistoryPass::floorAt(int tip) const
{
return tip > 0 ? std::min(startHeight_, tip) : startHeight_;
}
ShieldedHistoryPass::Work ShieldedHistoryPass::evaluate(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights,
int staleAfter, bool initialComplete) const
{
Work work;
work.total = addresses.size();
const int floor = active_ ? floorAt(tip) : -1;
std::size_t below = 0;
bool staleDue = false;
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it == heights.end()) {
++work.missing;
continue;
}
if (active_) {
if (it->second < floor) ++below;
} else if (tip > 0 && it->second < tip - staleAfter) {
staleDue = true;
}
}
work.pending = work.missing + (active_ ? below : (staleDue ? work.total - work.missing : 0));
work.explicitWork = !stalled() &&
(work.missing > 0 || (active_ ? (explicit_ && below > 0) : (staleDue && !initialComplete)));
work.routineWork = !work.explicitWork && work.pending > 0;
return work;
}
void ShieldedHistoryPass::begin(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights, int staleAfter,
bool initialComplete, bool routineAllowed)
{
if (active_) {
if (tip > 0 && startHeight_ > tip) startHeight_ = tip; // the chain reorganised below the floor
return;
}
if (tip <= 0 || !(routineAllowed || !initialComplete)) return;
bool staleDue = false;
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it != heights.end() && it->second < tip - staleAfter) {
staleDue = true;
break;
}
}
if (!staleDue) return;
active_ = true;
explicit_ = !initialComplete;
startHeight_ = tip;
}
int ShieldedHistoryPass::scanFloor(int tip, bool routineAllowed) const
{
return (active_ && (explicit_ || routineAllowed)) ? floorAt(tip) : -1;
}
void ShieldedHistoryPass::finish(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights, std::size_t attempted,
std::size_t succeeded)
{
if (attempted > 0) stalledCycles_ = succeeded > 0 ? 0 : stalledCycles_ + 1;
// An empty list proves nothing (a failed enumeration blanks it); a real wallet change resets the pass.
if (!active_ || addresses.empty()) return;
const int floor = floorAt(tip);
for (const auto& address : addresses) {
const auto it = heights.find(address);
if (it == heights.end() || it->second < floor) return; // still work left in this pass
}
active_ = false;
explicit_ = false;
}
void ShieldedHistoryPass::reset()
{
*this = ShieldedHistoryPass{};
}
TxCycle decideTxCycle(const TxCycleInputs& in)
{
const bool wanted = in.neverFetched || in.newTip || in.dirty || in.explicitScanWork || in.routineScanWork;
const bool scansAllowed = !in.syncing && in.heightKnown;
if (wanted) {
if (scansAllowed && (in.explicitScanWork || in.gateOpen)) return TxCycle::Full;
// Rows now (a mutation, a tracked send at a new tip); wallet-wide scans wait for the throttle.
if (in.neverFetched || in.dirty || (in.sendTracking && in.newTip)) return TxCycle::Light;
}
if (in.recentDue) return (scansAllowed && in.gateOpen) ? TxCycle::RecentWithProbe : TxCycle::Recent;
return TxCycle::None;
}
} // namespace services
} // namespace dragonx
src/services/history_scan_policy.h
+135
View File
@@ -0,0 +1,135 @@
// DragonX Wallet - ImGui Edition
// Copyright 2024-2026 The Hush Developers
// Released under the GPLv3
#pragma once
// When the transaction history refresh may scan what. The history comes from three wallet-wide RPCs, each
// holding the daemon's cs_main for its whole run (seconds per call on a large wallet): listtransactions,
// z_listreceivedbyaddress (once per z-address) and z_listsentbyaddress. This decides, per refresh cycle,
// which of them run, so the cost stays bounded and every pass over the z-addresses finishes. Pure and
// main-thread only; App::refreshTransactionData() feeds it and the test target drives it directly.
#include "rpc/rpc_failure.h"
#include <nlohmann/json.hpp>
#include <cstddef>
#include <string>
#include <unordered_map>
#include <vector>
namespace dragonx {
namespace services {
constexpr int kHistoryBlockSeconds = 36; // -ac_blocktime=36
constexpr double kHistoryScanDuty = 0.10; // the same 10% as the other scan throttles
constexpr int kHistoryMinStaleBlocks = 8; // re-scan an address at most every 8 blocks...
constexpr int kHistoryMaxStaleBlocks = 500; // ...and at least every 500 (a safety net)
constexpr int kSentSweepReorgMarginBlocks = 100; // > MAX_REORG_LENGTH (99), plus tip drift
constexpr long kFullSentSweepTimeoutSec = 120; // a whole-wallet sweep must be able to finish
constexpr int kTransparentPageMin = 1000;
constexpr int kTransparentPageMax = 5000;
constexpr int kStallCyclesBeforeGating = 1;
// Blocks after which an address's last receive scan counts as stale and a new routine pass is due: the
// pass then costs about kHistoryScanDuty of wall-clock (one z_listreceivedbyaddress of perAddressScanMs per
// address, spread over that many blocks), within [kHistoryMinStaleBlocks, kHistoryMaxStaleBlocks].
int historyStaleAfterBlocks(std::size_t addressCount, double perAddressScanMs);
// Running cost estimate: a higher sample replaces it at once, a lower one pulls it down 25%. A sample of 0
// or less (nothing measured) leaves it as it is.
double updateScanCostEstimateMs(double previousMs, double sampleMs);
// listtransactions rows to ask for in the first page: last time's count plus headroom, so most wallets
// get every row in one call instead of re-paging the whole history in 1000-row pages.
int transparentPageSize(std::size_t previousRowCount);
// z_listsentbyaddress: the whole wallet when there's no usable watermark (none yet, or it is above the
// tip after a reorg), else only txs at most (tip - fromHeight) + kSentSweepReorgMarginBlocks deep.
bool sentSweepIsFull(int tip, int fromHeight);
// [z, 0] for the full sweep, [z, 0, 2, maxDepth] for the window: JSON integers, and never 3 params (the
// daemon rejects that count).
nlohmann::json sentSweepParams(const std::string& anyZAddress, int tip, int fromHeight);
// Backoff after full sweeps that failed in a way that cost the daemon the work: 0, 10 min, 1 h, then 6 h.
long fullSentSweepRetryDelaySec(int consecutiveFailures);
// Whether a failed full sweep counts toward that backoff: only when the daemon did the work (reply timeout,
// an error answer, anything unclassified), not when the call never ran (held back, aborted, not listening).
bool fullSentSweepFailureCounts(rpc::RpcFailureKind kind);
// The address scanned longest ago (never-scanned first), for the between-blocks probe.
std::size_t oldestScannedAddressIndex(const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights);
// One pass of receive scans over every z-address. An address is done once it was scanned at or above the
// pass's start height (its floor), so a pass ends after ceil(A / budget) cycles however the tip moves. An
// explicit pass (the first load of a wallet, or after an invalidation) may run back to back; a routine
// re-pass (addresses gone stale) runs only as the scan throttle allows.
class ShieldedHistoryPass {
public:
struct Work {
std::size_t total = 0; // z-addresses
std::size_t missing = 0; // never scanned
std::size_t pending = 0; // still to scan in this pass (or in the pass that is due)
bool explicitWork = false; // may skip the scan throttle
bool routineWork = false; // waits for the scan throttle
};
Work evaluate(int tip, const std::vector<std::string>& addresses,
const std::unordered_map<std::string, int>& heights, int staleAfter, bool initialComplete) const;
// Start a pass if one is due and allowed (routine passes need routineAllowed, unless the first load
// isn't complete yet). Clamps an active pass's floor after a reorg below it.
void begin(int tip, const std::vector<std::string>& addresses, const std::unordered_map<std::string, int>& heights,
int staleAfter, bool initialComplete, bool routineAllowed);
// The floor a cycle scans up to: -1 means only never-scanned (and priority) addresses, which is all an
// explicit-only cycle may do while a routine pass is active.
int scanFloor(int tip, bool routineAllowed) const;
// After a cycle's heights are merged: count stalls (scans attempted, none succeeded), end the pass once
// no address is missing or below its floor.
void finish(int tip, const std::vector<std::string>& addresses, const std::unordered_map<std::string, int>& heights,
std::size_t attempted, std::size_t succeeded);
void reset();
bool active() const { return active_; }
bool explicitPass() const { return explicit_; }
int startHeight() const { return startHeight_; }
// Scans keep failing: explicit work is treated as routine (gated) until one succeeds again.
bool stalled() const { return stalledCycles_ >= kStallCyclesBeforeGating; }
private:
int floorAt(int tip) const;
bool active_ = false;
bool explicit_ = false;
int startHeight_ = -1;
int stalledCycles_ = 0;
};
enum class TxCycle {
None,
Full, // listtransactions, receive scans, the sent sweep, enrichment
Light, // listtransactions and enrichment only: no wallet-wide z-scans
Recent, // the between-blocks poll: recent listtransactions rows
RecentWithProbe, // ...plus one receive scan of the address scanned longest ago
};
struct TxCycleInputs {
bool neverFetched = false; // no history loaded yet this session
bool newTip = false; // the tip moved since the last history apply
bool dirty = false; // something changed the wallet (a send, an import, ...)
bool sendTracking = false; // tracked sends still waiting for their view
bool explicitScanWork = false; // ShieldedHistoryPass::Work::explicitWork
bool routineScanWork = false; // ShieldedHistoryPass::Work::routineWork
bool gateOpen = false; // the scan throttle allows a wallet-wide scan now
bool syncing = false; // the node is catching up (or settling after it)
bool heightKnown = false; // the tip height is known
bool recentDue = false; // the between-blocks poll is due on a wallet page
};
// Only explicit work skips the throttle with scans; sends and other mutations get rows now through Light
// cycles; nothing scans while syncing or before the tip is known. Tracking alone needs a new tip for a
// Light cycle, so it can't loop them at one tip.
TxCycle decideTxCycle(const TxCycleInputs& in);
} // namespace services
} // namespace dragonx
src/services/network_refresh_service.cpp
+230 -70
View File
@@ -1,4 +1,5 @@
#include "network_refresh_service.h"
#include "history_scan_policy.h"
#include "rpc/rpc_busy_gate.h"
#include "../util/logger.h"
@@ -134,20 +135,43 @@ void appendTrackedSendPlaceholder(std::vector<TransactionInfo>& transactions,
transactions.push_back(std::move(info));
}
// A row the shielded scans (or z_viewtransaction) produce rather than listtransactions.
bool isShieldedRelatedRow(const TransactionInfo& transaction)
{
return (!transaction.address.empty() && transaction.address[0] == 'z') ||
(!transaction.from_address.empty() && transaction.from_address[0] == 'z') ||
!transaction.memo.empty();
}
void appendMissingPreviousTransactions(std::vector<TransactionInfo>& transactions,
const std::vector<TransactionInfo>& previousTransactions,
bool includeAll,
const std::unordered_set<std::string>& pinnedTxids)
{
// Presence is decided against the rows built this cycle, before any is carried: deduping against rows
// just appended kept only the first leg of a multi-recipient send.
std::set<std::pair<std::string, std::string>> present;
for (const auto& transaction : transactions) present.emplace(transaction.txid, transaction.type);
for (const auto& transaction : previousTransactions) {
bool pinned = pinnedTxids.find(transaction.txid) != pinnedTxids.end();
if (!includeAll && !pinned) {
bool shieldedRelated = (!transaction.address.empty() && transaction.address[0] == 'z') ||
(!transaction.from_address.empty() && transaction.from_address[0] == 'z') ||
!transaction.memo.empty();
if (!shieldedRelated) continue;
}
if (hasTransactionType(transactions, transaction.txid, transaction.type)) continue;
if (!includeAll && !pinned && !isShieldedRelatedRow(transaction)) continue;
if (present.count({transaction.txid, transaction.type})) continue;
transactions.push_back(transaction);
}
}
// On a full rebuild, keep the stored rows of sends still waiting to be re-enriched. Presence is decided
// against the rows built this cycle, before any is carried, so every row of a multi-recipient send is kept
// (deduping against rows just appended would keep only the first).
void appendPinnedPreviousTransactions(std::vector<TransactionInfo>& transactions,
const std::vector<TransactionInfo>& previousTransactions,
const std::unordered_set<std::string>& pinnedTxids)
{
std::set<std::pair<std::string, std::string>> present;
for (const auto& transaction : transactions) present.emplace(transaction.txid, transaction.type);
for (const auto& transaction : previousTransactions) {
if (!pinnedTxids.count(transaction.txid)) continue;
if (present.count({transaction.txid, transaction.type})) continue;
transactions.push_back(transaction);
}
}
@@ -767,6 +791,9 @@ NetworkRefreshService::AddressRefreshResult NetworkRefreshService::collectAddres
lite.confirmations = (nz.contains("rawconfirmations") && nz["rawconfirmations"].is_number_integer())
? nz["rawconfirmations"].get<int>()
: nz.value("confirmations", 0);
if (nz.contains("address") && nz["address"].is_string()) lite.address = nz["address"].get<std::string>();
if (nz.contains("txid") && nz["txid"].is_string()) lite.txid = nz["txid"].get<std::string>();
if (nz.contains("change") && nz["change"].is_boolean()) lite.change = nz["change"].get<bool>();
result.unspentNotes.push_back(lite);
}
}
@@ -1044,10 +1071,13 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
result.blockHeight = currentBlockHeight;
result.shieldedAddressCount = snapshot.shieldedAddresses.size();
result.shieldedScanHeights = snapshot.shieldedScanHeights;
// Time the whole scan — the per-address z_listreceivedbyaddress pass is O(mapWallet) and holds the
// daemon's cs_main. The measured cost feeds the caller's adaptive throttle (txRefreshDue()) so the
// routine full history rescan can't starve block connection on a large wallet.
// Time the whole cycle: listtransactions, the per-address z_listreceivedbyaddress scans and
// z_listsentbyaddress are each O(mapWallet) and hold the daemon's cs_main. The measured cost feeds the
// caller's throttle (txRefreshDue()) so routine scans can't starve block connection on a large wallet.
const auto scanStart = std::chrono::steady_clock::now();
auto msSince = [](std::chrono::steady_clock::time_point from) {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - from).count();
};
std::set<std::string> knownTxids;
HushChatMemoOutputMap hushChatReceivedOutputs;
@@ -1056,47 +1086,62 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
: nullptr;
bool transactionRpcError = false;
try {
constexpr int pageSize = 1000;
int skip = 0;
while (true) {
json transactions = rpc.call("listtransactions", json::array({"", pageSize, skip}));
appendTransparentTransactions(result.transactions, knownTxids, transactions, snapshot.miningAddresses);
if (!transactions.is_array() || transactions.size() < static_cast<std::size_t>(pageSize)) break;
skip += pageSize;
// ── Transparent rows (listtransactions) ─────────────────────────────────────────────────────────
if (snapshot.reuseTransparentRows && !snapshot.previousTransactions.empty()) {
// Same tip and nothing changed since the rows on screen were fetched: keep their transparent rows
// instead of paging the whole history again. Shielded rows come from the scans below, and rows of
// tracked sends and pending ops are rebuilt by enrichment.
for (const auto& transaction : snapshot.previousTransactions) {
if (isShieldedRelatedRow(transaction)) continue;
if (snapshot.sendTxids.count(transaction.txid) || snapshot.pendingOpids.count(transaction.txid)) continue;
result.transactions.push_back(transaction);
if (!transaction.txid.empty()) knownTxids.insert(transaction.txid);
}
result.transparentRowsReused = true;
} else {
try {
// The first page holds last time's count plus headroom, so most wallets get every row in one
// call (re-paging 1000 rows at a time formats ~rows^2/2000 rows); skip-paging stays the fallback.
const int pageSize = transparentPageSize(snapshot.transparentRowHint);
for (int skip = 0;; skip += pageSize) {
json page = rpc.call("listtransactions", json::array({"", pageSize, skip}));
appendTransparentTransactions(result.transactions, knownTxids, page, snapshot.miningAddresses);
const std::size_t got = page.is_array() ? page.size() : 0;
result.transparentRowCount += got;
if (got < static_cast<std::size_t>(pageSize)) break;
}
result.transparentChainOk = true;
} catch (const std::exception& e) {
transactionRpcError = true;
DEBUG_LOGF("listtransactions error: %s\n", e.what());
}
} catch (const std::exception& e) {
transactionRpcError = true;
DEBUG_LOGF("listtransactions error: %s\n", e.what());
}
// An address counts as "scanned at tip" if its last scan is within shieldedScanTipTolerance
// blocks of the current tip. Tolerance 0 means strict (exact tip); a small tolerance lets a
// multi-cycle pass over many addresses complete even as the tip advances during it.
const int scanStaleThreshold = currentBlockHeight - std::max(0, snapshot.shieldedScanTipTolerance);
auto scannedAtTip = [&](const std::string& address) {
if (currentBlockHeight < 0) return false;
auto it = result.shieldedScanHeights.find(address);
return it != result.shieldedScanHeights.end() && it->second >= scanStaleThreshold;
// ── Shielded receives (z_listreceivedbyaddress, one per z-address) ──────────────────────────────
// A pass re-scans each address once: one scanned below the pass floor (or never) needs a scan, so a
// pass over many addresses completes however the tip moves meanwhile. Priority addresses go first.
const int scanFloor = snapshot.shieldedScanFloorHeight;
auto needsPassScan = [&](const std::string& address) {
const auto it = result.shieldedScanHeights.find(address);
if (it == result.shieldedScanHeights.end()) return true;
return scanFloor >= 0 && it->second < scanFloor;
};
std::size_t shieldedStart = snapshot.shieldedScanStartIndex;
if (shieldedStart >= snapshot.shieldedAddresses.size()) shieldedStart = 0;
std::size_t nextSearchIndex = shieldedStart;
for (std::size_t index = shieldedStart; index < snapshot.shieldedAddresses.size(); ++index) {
const auto& address = snapshot.shieldedAddresses[index];
if (scannedAtTip(address)) {
nextSearchIndex = index + 1;
continue;
}
if (snapshot.maxShieldedReceiveScans > 0 &&
result.shieldedAddressesScanned >= snapshot.maxShieldedReceiveScans) {
nextSearchIndex = index;
break;
}
auto needsPriorityScan = [&](const std::string& address) {
const auto priority = snapshot.priorityScanFloors.find(address);
if (priority == snapshot.priorityScanFloors.end()) return false;
const auto it = result.shieldedScanHeights.find(address);
return it == result.shieldedScanHeights.end() || it->second < priority->second;
};
auto budgetLeft = [&]() {
return snapshot.maxShieldedReceiveScans == 0 ||
result.shieldedAddressesScanned < snapshot.maxShieldedReceiveScans;
};
std::unordered_set<std::string> scannedNow;
auto scanAddress = [&](const std::string& address) {
++result.shieldedAddressesScanned;
nextSearchIndex = index + 1;
scannedNow.insert(address);
const auto callStart = std::chrono::steady_clock::now();
try {
json received = rpc.call("z_listreceivedbyaddress", json::array({address, 0}));
appendShieldedReceivedTransactions(result.transactions,
@@ -1105,11 +1150,36 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
received,
snapshot.miningAddresses,
hushChatReceivedOutputsPtr);
++result.shieldedScansSucceeded;
result.receiveScanOkMs += msSince(callStart);
if (currentBlockHeight >= 0) result.shieldedScanHeights[address] = currentBlockHeight;
} catch (const std::exception& e) {
transactionRpcError = true;
DEBUG_LOGF("z_listreceivedbyaddress error: %s\n", e.what());
}
};
std::size_t shieldedStart = snapshot.shieldedScanStartIndex;
if (shieldedStart >= snapshot.shieldedAddresses.size()) shieldedStart = 0;
std::size_t nextSearchIndex = shieldedStart;
if (!snapshot.skipShieldedScans) {
for (const auto& address : snapshot.shieldedAddresses) {
if (!budgetLeft()) break;
if (needsPriorityScan(address)) scanAddress(address);
}
for (std::size_t index = shieldedStart; index < snapshot.shieldedAddresses.size(); ++index) {
const auto& address = snapshot.shieldedAddresses[index];
if (!needsPassScan(address) || scannedNow.count(address)) {
nextSearchIndex = index + 1;
continue;
}
if (!budgetLeft()) {
nextSearchIndex = index;
break;
}
nextSearchIndex = index + 1;
scanAddress(address);
}
}
auto findPendingShieldedIndex = [&]() -> std::optional<std::size_t> {
@@ -1117,7 +1187,7 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
std::size_t start = nextSearchIndex < snapshot.shieldedAddresses.size() ? nextSearchIndex : 0;
for (std::size_t offset = 0; offset < snapshot.shieldedAddresses.size(); ++offset) {
std::size_t index = (start + offset) % snapshot.shieldedAddresses.size();
if (!scannedAtTip(snapshot.shieldedAddresses[index])) return index;
if (needsPassScan(snapshot.shieldedAddresses[index])) return index;
}
return std::nullopt;
};
@@ -1126,11 +1196,11 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
result.shieldedScanComplete = !pendingShieldedIndex.has_value();
result.nextShieldedScanStartIndex = pendingShieldedIndex.value_or(0);
// Discover outgoing shielded (z->z) sends. A pure z->z send moves no value in or out of the
// transparent pool (valueBalance == 0), so the daemon's `listtransactions` never reports it,
// and — being a send rather than a receive — it never shows up in `z_listreceivedbyaddress`
// either. Its txid therefore never reaches `knownTxids`, so the z_viewtransaction enrichment
// below is never run for it and the send is invisible in History (the reported bug).
// ── Outgoing shielded sends (z_listsentbyaddress) ───────────────────────────────────────────────
// A pure z->z send moves no value in or out of the transparent pool (valueBalance == 0), so the
// daemon's `listtransactions` never reports it, and — being a send rather than a receive — it never
// shows up in `z_listreceivedbyaddress` either. Its txid only reaches `knownTxids`, and so the
// z_viewtransaction enrichment below, through this call.
//
// `z_listsentbyaddress` scans the whole wallet for sends — the address argument is NOT used to
// filter (the daemon iterates every wallet tx against "*"), so a single call yields all send
@@ -1144,13 +1214,22 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
// deduped (by txid+address+amount) against listtransactions-derived t->t sends above, so a
// purely-transparent (t->t) send — which the payload ALSO reports — is not double-counted.
//
// Gated on shieldedScanComplete so it adds no cost during the multi-cycle initial received
// scan: it is one more O(mapWallet) pass holding cs_main (comparable to the listtransactions
// pass), and the caller's adaptive throttle (txRefreshDue) already accounts for it via scanMs.
if (result.shieldedScanComplete && !snapshot.shieldedAddresses.empty()) {
// It trial-decrypts every send it returns under cs_main, the heaviest call here on a wallet with many
// sends. So it runs once a pass is complete and, past a successful sweep (the caller's watermark), only
// over the txs at most a little deeper than the blocks since then (filter type 2). The whole-wallet
// sweep, needed when there's no watermark, gets a long timeout so it can finish on a large wallet.
const bool sweepIsFull = sentSweepIsFull(currentBlockHeight, snapshot.sentScanFromHeight);
if (!snapshot.shieldedAddresses.empty() && !snapshot.skipSentSweep &&
(result.shieldedScanComplete || (snapshot.allowMidPassSentSweep && !sweepIsFull))) {
result.sentSweepRan = true;
result.sentSweepFull = sweepIsFull;
const json params = sentSweepParams(snapshot.shieldedAddresses.front(), currentBlockHeight,
snapshot.sentScanFromHeight);
const auto sweepStart = std::chrono::steady_clock::now();
try {
json sent = rpc.call("z_listsentbyaddress",
json::array({snapshot.shieldedAddresses.front(), 0}));
json sent = result.sentSweepFull
? rpc.callLong("z_listsentbyaddress", params, kFullSentSweepTimeoutSec)
: rpc.call("z_listsentbyaddress", params);
if (sent.is_array()) {
for (const auto& sentTx : sent) {
auto txid = readOptional<std::string>(sentTx, "txid");
@@ -1158,11 +1237,15 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
appendSentTransparentOutputs(result.transactions, *txid, sentTx);
knownTxids.insert(*txid); // shielded (z->z) legs enriched via z_viewtransaction
}
result.sentSweepOk = true;
result.sentSweepHeight = currentBlockHeight;
}
} catch (const std::exception& e) {
transactionRpcError = true;
result.sentSweepErrKind = rpc::rpcFailureKindOf(e);
DEBUG_LOGF("z_listsentbyaddress error: %s\n", e.what());
}
result.sentSweepMs = msSince(sweepStart);
}
for (const auto& txid : snapshot.sendTxids) {
@@ -1173,17 +1256,41 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
if (!cachedView.outgoing_outputs.empty()) knownTxids.insert(txid);
}
// Shielded send rows from an earlier session (the history cache) whose view isn't cached: they count as
// enriched by their age, so nothing would rebuild them, and a windowed sweep never rediscovers an old
// send. Re-enrich them, and keep their stored rows ("pinned") until the daemon has answered.
std::unordered_set<std::string> rowBacklog;
// A settled stored row's block time and confirmations (>= 10, the History's "confirmed"): reused for
// its re-enrichment instead of a gettransaction per row. An unsettled one still asks, or its stale
// count would stick for the session (a cached view keeps the metadata it was built with).
std::unordered_map<std::string, std::pair<std::int64_t, int>> settledStoredMeta;
for (const auto& transaction : snapshot.previousTransactions) {
if (transaction.type != "send" || transaction.txid.empty() || !isShieldedRelatedRow(transaction)) continue;
if (snapshot.viewTxCache.count(transaction.txid) || snapshot.sendTxids.count(transaction.txid) ||
snapshot.pendingOpids.count(transaction.txid)) continue;
rowBacklog.insert(transaction.txid);
if (transaction.timestamp != 0 && transaction.confirmations >= 10)
settledStoredMeta.emplace(transaction.txid, std::make_pair(transaction.timestamp, transaction.confirmations));
}
std::set<std::string> backlog(snapshot.enrichBacklogTxids.begin(), snapshot.enrichBacklogTxids.end());
backlog.insert(rowBacklog.begin(), rowBacklog.end());
std::unordered_set<std::string> pinnedRows = rowBacklog;
// Tracked sends go first so they get the per-cycle z_viewtransaction budget ahead of routine
// enrichment, and a spent budget only skips further RPCs: cached rows later in the order are still
// appended (stopping the loop at the cap dropped them from History).
// enrichment, then this cycle's discoveries, then the backlog. A spent budget only skips further
// RPCs: cached rows later in the order are still appended (stopping the loop at the cap dropped them
// from History), and what was skipped is handed back as unenrichedTxids.
std::vector<const std::string*> enrichOrder;
enrichOrder.reserve(knownTxids.size());
enrichOrder.reserve(knownTxids.size() + backlog.size());
for (const auto& txid : knownTxids) {
if (snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid);
}
for (const auto& txid : knownTxids) {
if (!snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid);
}
for (const auto& txid : backlog) {
if (!knownTxids.count(txid) && !snapshot.sendTxids.count(txid)) enrichOrder.push_back(&txid);
}
int viewTxCount = 0;
for (const std::string* txidPtr : enrichOrder) {
@@ -1202,9 +1309,12 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
}
}
if (!trackedSend && snapshot.fullyEnrichedTxids.count(txid)) continue;
if (!trackedSend && snapshot.fullyEnrichedTxids.count(txid) && !rowBacklog.count(txid)) continue;
if (viewTxCount >= maxViewTransactionsPerCycle) continue;
if (viewTxCount >= maxViewTransactionsPerCycle) {
if (!trackedSend) result.unenrichedTxids.insert(txid);
continue;
}
++viewTxCount;
try {
@@ -1216,11 +1326,22 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
appendTrackedSendPlaceholder(result.transactions,
txid,
hasRawTransaction ? &rawTransaction : nullptr);
} else {
result.unenrichedTxids.insert(txid);
}
continue;
}
auto entry = parseViewTransactionCacheEntry(viewTransaction);
const bool storedRow = rowBacklog.count(txid) > 0;
if (storedRow && entry.timestamp == 0) {
const auto settled = settledStoredMeta.find(txid);
if (settled != settledStoredMeta.end()) {
entry.timestamp = settled->second.first;
entry.confirmations = settled->second.second;
}
}
const std::size_t rowsBeforeView = result.transactions.size();
appendViewTransactionOutputs(result.transactions, txid, entry);
// (Chat metadata is harvested only from RECEIVED notes, not our own outgoing memos — see
// the cached-view branch above.)
@@ -1246,6 +1367,21 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
}
}
if (storedRow) {
// A stored send rebuilt without a date (gettransaction failed): keep its stored rows (still
// pinned) and retry next cycle, rather than cache an undated view for the session.
bool undated = false;
for (std::size_t i = rowsBeforeView; i < result.transactions.size(); ++i) {
if (result.transactions[i].txid == txid && result.transactions[i].timestamp == 0) undated = true;
}
if (undated) {
result.transactions.resize(rowsBeforeView);
result.unenrichedTxids.insert(txid);
continue;
}
pinnedRows.erase(txid);
}
if (trackedSend && !hasTransactionType(result.transactions, txid, "send")) {
if (!hasRawTransaction) hasRawTransaction = tryFetchRawTransaction(rpc, txid, rawTransaction);
appendTrackedSendPlaceholder(result.transactions,
@@ -1275,13 +1411,20 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
}
}
} catch (const std::exception& e) {
(void)e;
if (trackedSend) {
json rawTransaction;
bool hasRawTransaction = tryFetchRawTransaction(rpc, txid, rawTransaction);
appendTrackedSendPlaceholder(result.transactions,
txid,
hasRawTransaction ? &rawTransaction : nullptr);
} else if (const auto* rpcError = dynamic_cast<const rpc::RpcError*>(&e);
rpcError && rpcError->code == -5) {
// "Invalid or non-wallet transaction id": definitive. Cache an empty view so it isn't
// asked again, and let a stored row for it go.
result.newViewTxEntries[txid] = TransactionViewCacheEntry{};
pinnedRows.erase(txid);
} else {
result.unenrichedTxids.insert(txid); // anything else may be transient: retry
}
}
}
@@ -1289,17 +1432,21 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectTr
appendExtractedHushChatMetadata(result.hushChatMetadata, hushChatReceivedOutputs);
if (!snapshot.previousTransactions.empty()) {
std::unordered_set<std::string> pinned = snapshot.pendingOpids;
pinned.insert(pinnedRows.begin(), pinnedRows.end());
if (transactionRpcError) {
appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, true, snapshot.pendingOpids);
appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, true, pinned);
} else if (snapshot.shieldedAddresses.empty() || !result.shieldedScanComplete ||
result.shieldedAddressesScanned < result.shieldedAddressCount) {
appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, false, snapshot.pendingOpids);
appendMissingPreviousTransactions(result.transactions, snapshot.previousTransactions, false, pinned);
} else if (!pinnedRows.empty()) {
// A full rebuild, except the stored rows of sends still waiting to be re-enriched.
appendPinnedPreviousTransactions(result.transactions, snapshot.previousTransactions, pinnedRows);
}
}
sortTransactionsNewestFirst(result.transactions);
result.scanMs = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - scanStart).count();
result.scanMs = msSince(scanStart);
return result;
}
@@ -1310,6 +1457,7 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe
int pageSize)
{
TransactionRefreshResult result;
const auto scanStart = std::chrono::steady_clock::now();
result.blockHeight = currentBlockHeight;
result.transactions = snapshot.previousTransactions;
result.shieldedAddressCount = snapshot.shieldedAddresses.size();
@@ -1360,9 +1508,12 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe
if (snapshot.maxShieldedReceiveScans > 0) {
shieldedLimit = std::min(shieldedLimit, snapshot.maxShieldedReceiveScans);
}
// Without the probe (the scan throttle is closed) the poll is listtransactions only.
if (snapshot.skipShieldedScans) shieldedLimit = 0;
for (std::size_t index = shieldedStart; index < shieldedStart + shieldedLimit; ++index) {
const auto& address = snapshot.shieldedAddresses[index];
const auto callStart = std::chrono::steady_clock::now();
try {
std::vector<TransactionInfo> scannedTransactions;
std::set<std::string> scannedTxids;
@@ -1384,18 +1535,27 @@ NetworkRefreshService::TransactionRefreshResult NetworkRefreshService::collectRe
}
if (currentBlockHeight >= 0) result.shieldedScanHeights[address] = currentBlockHeight;
++result.shieldedAddressesScanned;
++result.shieldedScansSucceeded;
result.receiveScanOkMs += std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - callStart).count();
} catch (const std::exception& e) {
++result.shieldedAddressesScanned;
DEBUG_LOGF("recent z_listreceivedbyaddress error: %s\n", e.what());
}
}
result.nextShieldedScanStartIndex =
(shieldedStart + shieldedLimit >= snapshot.shieldedAddresses.size()) ? 0 : shieldedStart + shieldedLimit;
if (snapshot.skipShieldedScans) {
result.nextShieldedScanStartIndex = shieldedStart;
} else {
result.nextShieldedScanStartIndex =
(shieldedStart + shieldedLimit >= snapshot.shieldedAddresses.size()) ? 0 : shieldedStart + shieldedLimit;
}
result.shieldedScanComplete = true;
appendExtractedHushChatMetadata(result.hushChatMetadata, hushChatReceivedOutputs);
sortTransactionsNewestFirst(result.transactions);
result.scanMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - scanStart).count();
return result;
}
src/services/network_refresh_service.h
+45 -10
View File
@@ -35,6 +35,13 @@ public:
virtual ~RefreshRpcGateway() = default;
virtual nlohmann::json call(const std::string& method,
const nlohmann::json& params) = 0;
// A call allowed to outlast the default timeout (a whole-wallet sweep on a large wallet).
// Gateways without a separate timeout just call().
virtual nlohmann::json callLong(const std::string& method, const nlohmann::json& params, long timeoutSec)
{
(void)timeoutSec;
return call(method, params);
}
};
enum class Job {
@@ -175,6 +182,9 @@ public:
double amount = 0.0; // note value, DRGX
bool locked = false; // tied up by an in-flight send
int confirmations = 0; // TRUE depth (rawconfirmations when present, else confirmations)
std::string address; // the z-address holding it
std::string txid; // the transaction that created it
bool change = false; // change from the wallet's own spend (not a receive)
};
struct AddressRefreshResult {
@@ -221,14 +231,25 @@ public:
std::set<std::string> miningAddresses;
std::unordered_map<std::string, int> shieldedScanHeights;
std::size_t shieldedScanStartIndex = 0;
std::size_t maxShieldedReceiveScans = 0;
// How many blocks the tip may advance past an address's last scan before it counts as stale
// and needs re-scanning. 0 = strict (must be scanned at the exact current tip). A small
// tolerance lets a multi-cycle pass over many shielded addresses COMPLETE even though new
// blocks arrive mid-pass — otherwise the "scanned at tip" bar moves every block and the scan
// (and the transactions_dirty_ flag it drives) never finishes. It also naturally throttles
// full rescans to roughly once per `tolerance` blocks.
int shieldedScanTipTolerance = 0;
std::size_t maxShieldedReceiveScans = 0; // receive scans this cycle; 0 = no limit
// The pass's floor (services::ShieldedHistoryPass): an address scanned below it is scanned again.
// -1: only never-scanned addresses (and priority ones) are.
int shieldedScanFloorHeight = -1;
// Scanned first, whatever the cursor, when never scanned or scanned below their floor: new
// addresses, the chat reply address, addresses holding notes the history doesn't show yet.
std::unordered_map<std::string, int> priorityScanFloors;
bool skipShieldedScans = false; // a Light cycle (or a probe-less recent poll): no receive scans
bool skipSentSweep = false;
int sentScanFromHeight = -1; // the sweep's watermark; -1 = sweep the whole wallet
std::size_t transparentRowHint = 0; // listtransactions rows last time: sizes the first page
// Same tip and nothing changed since the rows on screen were fetched: reuse their transparent rows
// instead of paging listtransactions again.
bool reuseTransparentRows = false;
std::unordered_set<std::string> enrichBacklogTxids; // discovered earlier, not enriched yet
// Run a windowed sent sweep even though the receive pass isn't complete (the caller rate-limits
// this), so untracked sends (from the Console, or another wallet on the same seed) don't wait for a
// whole pass. The whole-wallet sweep stays tied to a completed pass.
bool allowMidPassSentSweep = false;
};
struct TransactionRefreshResult {
@@ -245,9 +266,23 @@ public:
std::size_t shieldedAddressCount = 0;
std::unordered_map<std::string, int> shieldedScanHeights;
bool shieldedScanComplete = true;
// Wall-clock spent in the history scan (z_listreceivedbyaddress — O(mapWallet), holds cs_main).
// Lets the caller throttle the routine full rescan by its measured cost (txRefreshDue()).
// Wall-clock spent in the whole cycle: listtransactions, the receive scans and the sent sweep each
// hold cs_main. Lets the caller throttle wallet-wide scans by their measured cost (txRefreshDue()).
double scanMs = 0.0;
std::size_t shieldedScansSucceeded = 0; // shieldedAddressesScanned counts attempts
double receiveScanOkMs = 0.0; // time in the receive scans that succeeded
std::size_t transparentRowCount = 0; // rows the listtransactions chain returned
bool transparentChainOk = false;
bool transparentRowsReused = false;
bool sentSweepRan = false;
bool sentSweepOk = false;
bool sentSweepFull = false;
int sentSweepHeight = -1; // the tip a successful sweep covered
double sentSweepMs = 0.0; // time in the sweep (part of scanMs)
rpc::RpcFailureKind sentSweepErrKind = rpc::RpcFailureKind::None;
// Txids found but not enriched (over the per-cycle cap, or the view failed transiently): the caller
// keeps them and feeds them back as enrichBacklogTxids.
std::unordered_set<std::string> unenrichedTxids;
};
struct OperationStatusPollResult {
tests/test_phase4.cpp
+647 -6
View File
@@ -27,6 +27,7 @@
#include "rpc/rpc_failure.h"
#include "rpc/rpc_busy_gate.h"
#include "resources/embedded_resources.h"
#include "services/history_scan_policy.h"
#include "services/network_refresh_service.h"
#include "services/refresh_scheduler.h"
#include "services/note_consolidation.h"
@@ -695,6 +696,15 @@ public:
responses[method].push_back({nlohmann::json(), true, {}, std::move(exception)});
}
int longCalls = 0;
long lastLongTimeout = 0;
nlohmann::json callLong(const std::string& method, const nlohmann::json& params, long timeoutSec) override
{
++longCalls;
lastLongTimeout = timeoutSec;
return call(method, params); // recorded like any call, so the pinned call sequences still hold
}
nlohmann::json call(const std::string& method, const nlohmann::json& params) override
{
calls.push_back({method, params});
@@ -2129,15 +2139,15 @@ void testNetworkRefreshRpcCollectors()
EXPECT_EQ(pagedResult.transactions.size(), static_cast<size_t>(1001));
EXPECT_EQ(pagedResult.transactions[0].txid, std::string("paged-1000"));
// shieldedScanTipTolerance: an address scanned a few blocks below the tip is re-scanned when
// strict (tolerance 0) but SKIPPED within tolerance — this is what lets a multi-block pass
// complete (and unsticks the perpetual "refreshing history" state).
// The pass floor: an address scanned below it is scanned again; without a floor (no pass running)
// only never-scanned addresses are. A pass's floor is its start height, so it completes however far
// the tip moves meanwhile (this is what unsticks the perpetual "refreshing history" state).
{
Refresh::TransactionRefreshSnapshot strictSnapshot;
strictSnapshot.shieldedAddresses = {"zs-a", "zs-b"};
strictSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
strictSnapshot.maxShieldedReceiveScans = 4;
strictSnapshot.shieldedScanTipTolerance = 0; // strict: heights 100 are stale at tip 102
strictSnapshot.shieldedScanFloorHeight = 102; // a pass that started at 102: both below it
MockRefreshRpc strictRpc;
strictRpc.addResponse("listtransactions", json::array());
strictRpc.addResponse("z_listreceivedbyaddress", json::array());
@@ -2149,8 +2159,7 @@ void testNetworkRefreshRpcCollectors()
Refresh::TransactionRefreshSnapshot tolerantSnapshot;
tolerantSnapshot.shieldedAddresses = {"zs-a", "zs-b"};
tolerantSnapshot.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
tolerantSnapshot.maxShieldedReceiveScans = 4;
tolerantSnapshot.shieldedScanTipTolerance = 5; // 100 >= 102-5 -> still fresh
tolerantSnapshot.maxShieldedReceiveScans = 4; // no floor: no pass is running
MockRefreshRpc tolerantRpc;
tolerantRpc.addResponse("listtransactions", json::array());
// No z_listreceivedbyaddress responses: the addresses must be skipped (else call() throws).
@@ -2279,6 +2288,7 @@ void testNetworkRefreshRpcCollectors()
staleProgressSnapshot.shieldedAddresses = {"zs-current", "zs-stale", "zs-missing"};
staleProgressSnapshot.shieldedScanHeights = {{"zs-current", 500}, {"zs-stale", 499}};
staleProgressSnapshot.maxShieldedReceiveScans = 1;
staleProgressSnapshot.shieldedScanFloorHeight = 500;
staleProgressSnapshot.previousTransactions.push_back(dragonx::TransactionInfo{
"current-shielded", "receive", 0.75, 110, 12, "zs-current", "", ""
});
@@ -9840,6 +9850,632 @@ void testScopeScanAndLockedNotes()
EXPECT_FALSE(noteIsLocked(json::array()));
}
// C43/C53/C52: the pure history-scan policy (services/history_scan_policy).
void testHistoryScanPolicy()
{
using namespace dragonx::services;
using dragonx::rpc::RpcFailureKind;
// A routine re-pass is due after enough blocks that it costs ~10% of wall-clock, within [8, 500].
EXPECT_EQ(historyStaleAfterBlocks(0, 0.0), 8);
EXPECT_EQ(historyStaleAfterBlocks(4, 500.0), 8); // 2 s of scans: the floor
EXPECT_EQ(historyStaleAfterBlocks(30, 10000.0), 84); // 300 s / (36 s * 10%) = 83.3
EXPECT_EQ(historyStaleAfterBlocks(1000, 60000.0), 500); // capped
EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits<double>::quiet_NaN()), 8);
EXPECT_EQ(historyStaleAfterBlocks(30, std::numeric_limits<double>::infinity()), 500);
// The per-address cost estimate rises at once and decays 25% per sample; no sample, no change.
EXPECT_NEAR(updateScanCostEstimateMs(0.0, 400.0), 400.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(400.0, 800.0), 800.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(800.0, 400.0), 700.0, 1e-9);
EXPECT_NEAR(updateScanCostEstimateMs(800.0, 0.0), 800.0, 1e-9);
// listtransactions' first page: last time's rows plus headroom, within [1000, 5000].
EXPECT_EQ(transparentPageSize(0), 1000);
EXPECT_EQ(transparentPageSize(900), 1100);
EXPECT_EQ(transparentPageSize(1500), 1700);
EXPECT_EQ(transparentPageSize(4800), 5000);
EXPECT_EQ(transparentPageSize(100000), 5000);
// The sent sweep: the whole wallet without a usable watermark, else a depth window. Never 3 params.
EXPECT_TRUE(sentSweepIsFull(0, 50));
EXPECT_TRUE(sentSweepIsFull(100, -1));
EXPECT_TRUE(sentSweepIsFull(100, 101)); // a watermark above the tip (reorg)
EXPECT_FALSE(sentSweepIsFull(100, 100));
EXPECT_EQ(sentSweepParams("zs-one", 100, -1).dump(), std::string("[\"zs-one\",0]"));
const auto windowed = sentSweepParams("zs-one", 110, 100);
EXPECT_EQ(windowed.dump(), std::string("[\"zs-one\",0,2,110]"));
EXPECT_TRUE(windowed.size() == 4 && windowed[1].is_number_integer() && windowed[2].is_number_integer() &&
windowed[3].is_number_integer());
// Backoff after whole-wallet sweeps that failed after the daemon did the work.
EXPECT_EQ(fullSentSweepRetryDelaySec(0), 0L);
EXPECT_EQ(fullSentSweepRetryDelaySec(1), 600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(2), 3600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(3), 21600L);
EXPECT_EQ(fullSentSweepRetryDelaySec(9), 21600L);
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::ReplyTimeout));
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::AppError));
EXPECT_TRUE(fullSentSweepFailureCounts(RpcFailureKind::Other));
for (auto kind : {RpcFailureKind::Deferred, RpcFailureKind::Aborted, RpcFailureKind::NotListening,
RpcFailureKind::QueueFull, RpcFailureKind::Warmup}) {
EXPECT_FALSE(fullSentSweepFailureCounts(kind));
}
// The between-blocks probe takes the address scanned longest ago, never-scanned first.
EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"b", 40}, {"c", 60}}), static_cast<size_t>(1));
EXPECT_EQ(oldestScannedAddressIndex({"a", "b", "c"}, {{"a", 50}, {"c", 60}}), static_cast<size_t>(1));
EXPECT_EQ(oldestScannedAddressIndex({}, {}), static_cast<size_t>(0));
// decideTxCycle.
auto cycle = [](void (*setup)(TxCycleInputs&)) {
TxCycleInputs in;
in.heightKnown = true;
setup(in);
return decideTxCycle(in);
};
EXPECT_TRUE(cycle([](TxCycleInputs&) {}) == TxCycle::None);
// A new block: a Full cycle only when the throttle allows; otherwise it waits.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.gateOpen = true; }) == TxCycle::Full);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; }) == TxCycle::None);
// Explicit scan work skips the throttle; routine work waits for it.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; }) == TxCycle::Full);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; }) == TxCycle::None);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.routineScanWork = true; in.gateOpen = true; }) == TxCycle::Full);
// A mutation or the first load gets rows now through a Light cycle (no wallet-wide z-scans).
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.dirty = true; }) == TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.neverFetched = true; }) == TxCycle::Light);
// Tracking a send alone needs a new tip for a Light cycle, so it can't loop them at one tip.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; in.newTip = true; }) == TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.sendTracking = true; }) == TxCycle::None);
// Nothing scans while syncing or before the tip is known, explicit work included.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.dirty = true; in.syncing = true; }) ==
TxCycle::Light);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.explicitScanWork = true; in.heightKnown = false; }) == TxCycle::None);
// The between-blocks poll probes one address only when the throttle allows.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; }) == TxCycle::Recent);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; }) == TxCycle::RecentWithProbe);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.recentDue = true; in.gateOpen = true; in.syncing = true; }) ==
TxCycle::Recent);
// At a new tip the cheap poll runs only while the throttle holds the Full cycle back.
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; }) == TxCycle::Recent);
EXPECT_TRUE(cycle([](TxCycleInputs& in) { in.newTip = true; in.recentDue = true; in.gateOpen = true; }) ==
TxCycle::Full);
}
namespace {
// Answers every call with an empty array and records the methods (for simulations).
class RecordingRefreshRpc : public dragonx::services::NetworkRefreshService::RefreshRpcGateway {
public:
std::vector<std::string> methods;
nlohmann::json call(const std::string& method, const nlohmann::json&) override
{
methods.push_back(method);
return nlohmann::json::array();
}
};
} // namespace
// C43 end to end: the policy, the pass and the real collector over 6 virtual hours on a large wallet (40
// z-addresses at 10 s per wallet-wide call, 36 s blocks, the Overview cadence), mirroring the App glue in
// refreshTransactionData / startTransactionCycle. The first load completes, routine passes keep
// completing, and the scans hold cs_main for at most ~10% of the time once the first load is done.
void testHistoryDutyModel()
{
using Refresh = dragonx::services::NetworkRefreshService;
using namespace dragonx::services;
constexpr int kAddresses = 40;
constexpr std::size_t kBudget = 4;
constexpr double kCallSec = 10.0;
constexpr double kTickSec = 10.0;
constexpr double kBlockSec = 36.0;
constexpr double kEndSec = 6 * 3600.0;
constexpr double kLateSec = 2 * 3600.0;
std::vector<std::string> addrs;
for (int i = 0; i < kAddresses; ++i) addrs.push_back("zs-d" + std::to_string(i));
std::unordered_map<std::string, int> heights;
ShieldedHistoryPass pass;
bool initialComplete = false;
int lastTxTip = -1, watermark = -1;
std::size_t cursor = 0;
double lastScanAt = -1.0, lastScanCost = 0.0, busyUntil = 0.0, lateScanSec = 0.0;
int routinePassesDone = 0;
for (double now = 0.0; now < kEndSec; now += kTickSec) {
if (now < busyUntil) continue; // one cycle in flight at a time
const int tip = 1000 + static_cast<int>(now / kBlockSec);
const int staleAfter = historyStaleAfterBlocks(addrs.size(), kCallSec * 1000.0);
const auto work = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete);
TxCycleInputs in;
in.neverFetched = lastTxTip < 0;
in.newTip = tip != lastTxTip;
in.explicitScanWork = work.explicitWork;
in.routineScanWork = work.routineWork;
in.gateOpen = lastScanAt < 0.0 || now - lastScanAt >= lastScanCost / kHistoryScanDuty;
in.heightKnown = true;
const TxCycle cycle = decideTxCycle(in);
if (cycle != TxCycle::Full && cycle != TxCycle::Light) continue;
const bool full = cycle == TxCycle::Full;
const bool routineAllowed = in.gateOpen;
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = addrs;
snap.shieldedScanHeights = heights;
if (full) {
const bool wasActive = pass.active();
pass.begin(tip, addrs, heights, staleAfter, initialComplete, routineAllowed);
if (!wasActive && pass.active()) cursor = 0;
snap.shieldedScanFloorHeight = pass.scanFloor(tip, routineAllowed);
snap.shieldedScanStartIndex = cursor;
snap.maxShieldedReceiveScans = kBudget;
for (const auto& a : addrs) {
if (!heights.count(a)) snap.priorityScanFloors[a] = tip;
}
snap.sentScanFromHeight = watermark;
snap.skipSentSweep = (!routineAllowed && initialComplete) || (watermark >= 0 && watermark == tip);
snap.allowMidPassSentSweep = routineAllowed && watermark >= 0 && watermark < tip &&
tip - watermark >= std::max(kHistoryMinStaleBlocks, staleAfter / 4);
} else {
snap.skipShieldedScans = true;
snap.skipSentSweep = true;
}
RecordingRefreshRpc rpc;
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 25);
double cost = 0.0, sweepCost = 0.0;
for (const auto& m : rpc.methods) {
if (m == "z_listreceivedbyaddress") cost += kCallSec;
if (m == "z_listsentbyaddress") { cost += kCallSec; sweepCost += kCallSec; }
}
busyUntil = now + cost;
if (now >= kLateSec) lateScanSec += cost;
lastTxTip = tip;
if (!full) continue;
lastScanCost = (result.sentSweepRan && result.sentSweepFull) ? std::max(0.001, cost - sweepCost) : cost;
lastScanAt = now + cost;
const bool wasActive = pass.active();
const bool wasExplicit = pass.explicitPass();
heights = result.shieldedScanHeights;
cursor = result.nextShieldedScanStartIndex;
if (result.sentSweepOk) watermark = result.sentSweepHeight;
pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded);
if (wasActive && !pass.active() && !wasExplicit) ++routinePassesDone;
const auto after = pass.evaluate(tip, addrs, heights, staleAfter, initialComplete);
if (after.total > 0 && after.missing == 0 && !(pass.active() && pass.explicitPass())) initialComplete = true;
}
EXPECT_TRUE(initialComplete);
EXPECT_TRUE(routinePassesDone >= 3);
const double share = lateScanSec / (kEndSec - kLateSec);
EXPECT_TRUE(share <= 0.12);
EXPECT_TRUE(share >= 0.05); // and it does keep scanning (a stuck model would read 0)
}
// C43: a pass re-scans every z-address once, ending when each is at or above the pass's start height.
void testShieldedHistoryPass()
{
using namespace dragonx::services;
const std::vector<std::string> addrs = {"z1", "z2", "z3"};
ShieldedHistoryPass pass;
// First load: nothing scanned yet, all of it explicit work.
auto work = pass.evaluate(100, addrs, {}, 8, false);
EXPECT_EQ(work.total, static_cast<size_t>(3));
EXPECT_EQ(work.missing, static_cast<size_t>(3));
EXPECT_EQ(work.pending, static_cast<size_t>(3));
EXPECT_TRUE(work.explicitWork);
EXPECT_FALSE(work.routineWork);
// All scanned at 100: nothing is due until they're older than staleAfter.
std::unordered_map<std::string, int> heights{{"z1", 100}, {"z2", 100}, {"z3", 100}};
work = pass.evaluate(105, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(0));
EXPECT_FALSE(work.explicitWork);
EXPECT_FALSE(work.routineWork);
// Stale: a routine pass is due, and doesn't start without the throttle.
work = pass.evaluate(109, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(3));
EXPECT_FALSE(work.explicitWork);
EXPECT_TRUE(work.routineWork);
pass.begin(109, addrs, heights, 8, true, false);
EXPECT_FALSE(pass.active());
EXPECT_EQ(pass.scanFloor(109, false), -1);
pass.begin(109, addrs, heights, 8, true, true);
EXPECT_TRUE(pass.active());
EXPECT_FALSE(pass.explicitPass());
EXPECT_EQ(pass.startHeight(), 109);
// A routine pass scans only in cycles the throttle allows.
EXPECT_EQ(pass.scanFloor(110, true), 109);
EXPECT_EQ(pass.scanFloor(110, false), -1);
// It ends once every address is at or above its floor, however far the tip has moved.
heights["z1"] = 112;
heights["z2"] = 115;
pass.finish(115, addrs, heights, 2, 2);
EXPECT_TRUE(pass.active()); // z3 is still below 109
work = pass.evaluate(115, addrs, heights, 8, true);
EXPECT_EQ(work.pending, static_cast<size_t>(1));
EXPECT_TRUE(work.routineWork);
pass.finish(140, {}, heights, 4, 0); // a list blanked by a failed enumeration proves nothing
EXPECT_TRUE(pass.active());
heights["z3"] = 140;
pass.finish(140, addrs, heights, 1, 1);
EXPECT_FALSE(pass.active());
// Before the first load completes, a pass is explicit: it may run back to back.
ShieldedHistoryPass first;
const std::unordered_map<std::string, int> cached{{"z1", 10}, {"z2", 10}, {"z3", 10}}; // the history cache
first.begin(200, addrs, cached, 8, false, false);
EXPECT_TRUE(first.active());
EXPECT_TRUE(first.explicitPass());
EXPECT_EQ(first.scanFloor(200, false), 200);
EXPECT_TRUE(first.evaluate(201, addrs, cached, 8, false).explicitWork);
first.begin(150, addrs, cached, 8, false, false); // a reorg below the floor pulls it down
EXPECT_EQ(first.startHeight(), 150);
first.reset();
EXPECT_FALSE(first.active());
EXPECT_EQ(first.startHeight(), -1);
// Scans that keep failing: explicit work turns routine (throttled) until one succeeds.
ShieldedHistoryPass stall;
stall.finish(100, addrs, {}, 2, 0);
EXPECT_TRUE(stall.stalled());
work = stall.evaluate(100, addrs, {}, 8, false);
EXPECT_FALSE(work.explicitWork);
EXPECT_TRUE(work.routineWork);
stall.finish(100, addrs, {}, 1, 1);
EXPECT_FALSE(stall.stalled());
EXPECT_TRUE(stall.evaluate(100, addrs, {}, 8, false).explicitWork);
}
// C43 with the real collector: a pass over 10 addresses, 4 per cycle, finishes in 3 cycles although the tip
// moves 5 blocks per cycle (with the old "scanned within T of the tip" rule addresses went stale again
// mid-pass and a large wallet's pass never finished), and the sent sweep runs once, when it's done.
void testHistoryPassWithCollector()
{
using Refresh = dragonx::services::NetworkRefreshService;
using namespace dragonx::services;
using json = nlohmann::json;
std::vector<std::string> addrs;
std::unordered_map<std::string, int> heights;
for (int i = 0; i < 10; ++i) {
addrs.push_back("zs-p" + std::to_string(i));
heights[addrs.back()] = 100;
}
ShieldedHistoryPass pass;
int tip = 120;
pass.begin(tip, addrs, heights, 8, true, true);
EXPECT_TRUE(pass.active());
std::size_t cursor = 0;
std::size_t scans = 0;
int cycles = 0, sweeps = 0, sweepCycle = -1;
while (pass.active() && cycles < 10) {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = addrs;
snap.shieldedScanHeights = heights;
snap.maxShieldedReceiveScans = 4;
snap.shieldedScanFloorHeight = pass.scanFloor(tip, true);
snap.shieldedScanStartIndex = cursor;
snap.sentScanFromHeight = 100;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
for (int i = 0; i < 4; ++i) rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, tip, 0);
++cycles;
scans += result.shieldedAddressesScanned;
for (const auto& call : rpc.calls) {
if (call.method == "z_listsentbyaddress") { ++sweeps; sweepCycle = cycles; }
}
heights = result.shieldedScanHeights;
cursor = result.nextShieldedScanStartIndex;
pass.finish(tip, addrs, heights, result.shieldedAddressesScanned, result.shieldedScansSucceeded);
tip += 5;
}
EXPECT_FALSE(pass.active());
EXPECT_EQ(cycles, 3);
EXPECT_EQ(scans, static_cast<size_t>(10));
EXPECT_EQ(sweeps, 1);
EXPECT_EQ(sweepCycle, 3);
// A new address while a routine pass is active and the throttle is closed: the cycle scans only it
// (priority, floor -1), not the routine pass's addresses, and the pass stays active.
std::vector<std::string> many;
std::unordered_map<std::string, int> manyHeights;
for (int i = 0; i < 12; ++i) {
many.push_back("zs-r" + std::to_string(i));
if (i != 2) manyHeights[many.back()] = 90;
}
ShieldedHistoryPass routine;
routine.begin(100, many, manyHeights, 8, true, true);
EXPECT_TRUE(routine.active());
Refresh::TransactionRefreshSnapshot explicitOnly;
explicitOnly.shieldedAddresses = many;
explicitOnly.shieldedScanHeights = manyHeights;
explicitOnly.maxShieldedReceiveScans = 4;
explicitOnly.shieldedScanFloorHeight = routine.scanFloor(105, false);
explicitOnly.shieldedScanStartIndex = 8;
explicitOnly.priorityScanFloors = {{many[2], 105}};
explicitOnly.skipSentSweep = true;
EXPECT_EQ(explicitOnly.shieldedScanFloorHeight, -1);
MockRefreshRpc newAddrRpc;
newAddrRpc.addResponse("listtransactions", json::array());
newAddrRpc.addResponse("z_listreceivedbyaddress", json::array());
auto newAddr = Refresh::collectTransactionRefreshResult(newAddrRpc, explicitOnly, 105, 0);
EXPECT_TRUE(newAddrRpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listreceivedbyaddress"}));
EXPECT_EQ(newAddrRpc.calls[1].params, json::array({many[2], 0}));
EXPECT_EQ(newAddr.shieldedAddressesScanned, static_cast<size_t>(1));
routine.finish(105, many, newAddr.shieldedScanHeights, 1, 1);
EXPECT_TRUE(routine.active());
}
// C53/C52: the sent sweep, enrichment backlog and carry-forward, the adaptive page and row reuse.
void testHistoryCollectorSweepAndEnrichment()
{
using Refresh = dragonx::services::NetworkRefreshService;
using json = nlohmann::json;
using dragonx::rpc::RpcFailureKind;
using dragonx::rpc::RpcTransportError;
using dragonx::rpc::RpcError;
auto freshSnapshot = []() {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-one"};
snap.shieldedScanHeights = {{"zs-one", 110}};
return snap;
};
auto countRows = [](const Refresh::TransactionRefreshResult& r, const std::string& txid) {
return static_cast<int>(std::count_if(r.transactions.begin(), r.transactions.end(),
[&](const dragonx::TransactionInfo& t) { return t.txid == txid; }));
};
{ // Past a watermark: a windowed sweep, an ordinary call.
auto snap = freshSnapshot();
snap.sentScanFromHeight = 100;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions", "z_listsentbyaddress"}));
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0,2,110]"));
EXPECT_EQ(rpc.longCalls, 0);
EXPECT_TRUE(result.sentSweepRan && result.sentSweepOk && !result.sentSweepFull);
EXPECT_EQ(result.sentSweepHeight, 110);
}
{ // No watermark: the whole-wallet sweep, given the long timeout.
auto snap = freshSnapshot();
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[1].params.dump(), std::string("[\"zs-one\",0]"));
EXPECT_EQ(rpc.longCalls, 1);
EXPECT_EQ(rpc.lastLongTimeout, 120L);
EXPECT_TRUE(result.sentSweepFull && result.sentSweepOk);
}
{ // A failed sweep reports why (the caller's backoff), and keeps every previous row.
auto snap = freshSnapshot();
snap.previousTransactions.push_back(dragonx::TransactionInfo{"r-old", "receive", 1.0, 50, 9, "R-one", "", ""});
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addException("z_listsentbyaddress", std::make_exception_ptr(RpcTransportError(
RpcFailureKind::ReplyTimeout, "RPC request failed: Timeout was reached", CURLE_OPERATION_TIMEDOUT, 0, true)));
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(result.sentSweepRan && !result.sentSweepOk);
EXPECT_TRUE(result.sentSweepErrKind == RpcFailureKind::ReplyTimeout);
EXPECT_EQ(countRows(result, "r-old"), 1);
}
{ // skipSentSweep: no sweep at all.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_FALSE(result.sentSweepRan);
}
{ // A Light cycle: no receive scans, no sweep; the shielded rows on screen are carried.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a"};
snap.skipShieldedScans = true;
snap.skipSentSweep = true;
snap.previousTransactions.push_back(dragonx::TransactionInfo{"s-old", "receive", 2.0, 60, 4, "zs-a", "", ""});
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_EQ(countRows(result, "s-old"), 1);
}
{ // Priority addresses are scanned before the cursor's.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a", "zs-b", "zs-c", "zs-d"};
snap.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}, {"zs-c", 100}, {"zs-d", 100}};
snap.shieldedScanFloorHeight = 105;
snap.priorityScanFloors = {{"zs-d", 105}};
snap.maxShieldedReceiveScans = 2;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
Refresh::collectTransactionRefreshResult(rpc, snap, 105, 4);
EXPECT_EQ(rpc.calls[1].params, json::array({"zs-d", 0}));
EXPECT_EQ(rpc.calls[2].params, json::array({"zs-a", 0}));
}
{ // Enrichment order: this cycle's discoveries, then the backlog; past the cap they're handed back.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.enrichBacklogTxids = {"b1"};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array({
json{{"txid", "t1"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 1.0}, {"time", 70}, {"confirmations", 0}},
json{{"txid", "t2"}, {"category", "receive"}, {"address", "R-one"}, {"amount", 2.0}, {"time", 71}, {"confirmations", 0}}
}));
rpc.addResponse("z_viewtransaction", json{{"spends", json::array()}, {"outputs", json::array()}});
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 1);
EXPECT_EQ(rpc.calls[1].params, json::array({"t1"}));
EXPECT_EQ(result.unenrichedTxids.size(), static_cast<size_t>(2));
EXPECT_EQ(result.unenrichedTxids.count("t2"), static_cast<size_t>(1));
EXPECT_EQ(result.unenrichedTxids.count("b1"), static_cast<size_t>(1));
}
// A shielded send row from the history cache (an earlier session) with no cached view: re-enriched.
// The cycles below are full rebuilds (every address scanned, the pass complete), which drop rows that
// aren't rebuilt, except the "pinned" ones still waiting for their view.
auto rebuildSnapshot = [](std::vector<dragonx::TransactionInfo> previous) {
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-mine"};
snap.shieldedScanHeights = {{"zs-mine", 100}};
snap.shieldedScanFloorHeight = 110;
snap.skipSentSweep = true;
snap.previousTransactions = std::move(previous);
return snap;
};
const dragonx::TransactionInfo oldSend{"zz-old", "send", -1.5, 900, 30, "zs-peer", "zs-mine", ""};
{ // The daemon answers: the row is rebuilt from the view, dated from the settled stored row (no
// gettransaction per row).
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_viewtransaction", json{
{"spends", json::array({json{{"address", "zs-mine"}}})},
{"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.5}}})}});
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({
"listtransactions", "z_listreceivedbyaddress", "z_viewtransaction"}));
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-old"), 1);
for (const auto& t : result.transactions) {
if (t.txid != "zz-old") continue;
EXPECT_EQ(t.timestamp, static_cast<std::int64_t>(900));
EXPECT_EQ(t.confirmations, 30);
}
}
{ // A stored unsettled send still asks gettransaction; if that fails, the stored row stays (pinned),
// no undated view is cached, and it is retried.
const dragonx::TransactionInfo pendingSend{"zz-pending", "send", -1.0, 950, 0, "zs-peer", "zs-mine", ""};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_viewtransaction", json{
{"spends", json::array({json{{"address", "zs-mine"}}})},
{"outputs", json::array({json{{"outgoing", true}, {"address", "zs-peer"}, {"value", 1.0}}})}});
rpc.addFailure("gettransaction", "busy");
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({pendingSend}), 110, 4);
EXPECT_EQ(rpc.calls.back().method, std::string("gettransaction"));
EXPECT_EQ(result.newViewTxEntries.count("zz-pending"), static_cast<size_t>(0));
EXPECT_EQ(result.unenrichedTxids.count("zz-pending"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-pending"), 1);
for (const auto& t : result.transactions) {
if (t.txid == "zz-pending") EXPECT_EQ(t.timestamp, static_cast<std::int64_t>(950));
}
}
{ // Over the cap: still pinned, so the stored row survives the rebuild.
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 0);
EXPECT_EQ(countRows(result, "zz-old"), 1);
EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast<size_t>(1));
}
{ // Only "not a wallet tx" (-5) is definitive: cached empty, and the row is let go.
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-5, "Invalid or non-wallet transaction id")));
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(1));
EXPECT_TRUE(result.newViewTxEntries.at("zz-old").outgoing_outputs.empty());
EXPECT_EQ(countRows(result, "zz-old"), 0);
}
for (int plainError = 0; plainError < 2; ++plainError) { // anything else: retried, the row kept
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
if (plainError) rpc.addFailure("z_viewtransaction", "boom");
else rpc.addException("z_viewtransaction", std::make_exception_ptr(RpcError(-1, "something else")));
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({oldSend}), 110, 4);
EXPECT_EQ(result.newViewTxEntries.count("zz-old"), static_cast<size_t>(0));
EXPECT_EQ(result.unenrichedTxids.count("zz-old"), static_cast<size_t>(1));
EXPECT_EQ(countRows(result, "zz-old"), 1);
}
{ // A pinned send to two recipients keeps both rows (not just the first per txid), in a full rebuild...
const dragonx::TransactionInfo legA{"zz-two", "send", -1.0, 900, 30, "zs-peer-a", "zs-mine", ""};
const dragonx::TransactionInfo legB{"zz-two", "send", -2.0, 900, 30, "zs-peer-b", "zs-mine", ""};
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, rebuildSnapshot({legA, legB}), 110, 0);
EXPECT_EQ(countRows(result, "zz-two"), 2);
// ...and in a cycle that carries rows (here a Light cycle).
Refresh::TransactionRefreshSnapshot light;
light.shieldedAddresses = {"zs-mine"};
light.shieldedScanHeights = {{"zs-mine", 100}};
light.skipShieldedScans = true;
light.skipSentSweep = true;
light.previousTransactions = {legA, legB};
MockRefreshRpc lightRpc;
lightRpc.addResponse("listtransactions", json::array());
auto carried = Refresh::collectTransactionRefreshResult(lightRpc, light, 110, 0);
EXPECT_EQ(countRows(carried, "zz-two"), 2);
EXPECT_EQ(carried.unenrichedTxids.count("zz-two"), static_cast<size_t>(1));
}
{ // A windowed sweep mid-pass when allowed (rate-limited by the caller); never the whole-wallet one.
Refresh::TransactionRefreshSnapshot midPass;
midPass.shieldedAddresses = {"zs-a", "zs-b"};
midPass.shieldedScanHeights = {{"zs-a", 100}, {"zs-b", 100}};
midPass.shieldedScanFloorHeight = 110;
midPass.maxShieldedReceiveScans = 1;
midPass.sentScanFromHeight = 100;
midPass.allowMidPassSentSweep = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
rpc.addResponse("z_listreceivedbyaddress", json::array());
rpc.addResponse("z_listsentbyaddress", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, midPass, 110, 4);
EXPECT_FALSE(result.shieldedScanComplete);
EXPECT_TRUE(result.sentSweepRan && !result.sentSweepFull);
EXPECT_EQ(rpc.calls.back().params.dump(), std::string("[\"zs-a\",0,2,110]"));
midPass.allowMidPassSentSweep = false;
MockRefreshRpc offRpc;
offRpc.addResponse("listtransactions", json::array());
offRpc.addResponse("z_listreceivedbyaddress", json::array());
EXPECT_FALSE(Refresh::collectTransactionRefreshResult(offRpc, midPass, 110, 4).sentSweepRan);
midPass.allowMidPassSentSweep = true;
midPass.sentScanFromHeight = -1; // no watermark: the whole-wallet sweep waits for the pass
MockRefreshRpc fullRpc;
fullRpc.addResponse("listtransactions", json::array());
fullRpc.addResponse("z_listreceivedbyaddress", json::array());
EXPECT_FALSE(Refresh::collectTransactionRefreshResult(fullRpc, midPass, 110, 4).sentSweepRan);
}
{ // C52: the first page is sized from last time's rows.
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.transparentRowHint = 1500;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[0].params, json::array({"", 1700, 0}));
EXPECT_TRUE(result.transparentChainOk);
}
{ // Same tip, nothing changed: the transparent rows are reused, no listtransactions at all; a tracked
// send is rebuilt once (not reused and placeheld twice).
auto snap = freshSnapshot();
snap.skipSentSweep = true;
snap.reuseTransparentRows = true;
snap.sendTxids = {"t-sent"};
snap.previousTransactions = {
dragonx::TransactionInfo{"r1", "receive", 1.0, 80, 5, "R-one", "", ""},
dragonx::TransactionInfo{"s1", "receive", 2.0, 81, 5, "zs-one", "", ""},
dragonx::TransactionInfo{"t-sent", "send", -0.5, 82, 0, "R-two", "R-one", ""},
};
MockRefreshRpc rpc;
rpc.addResponse("z_viewtransaction", json());
rpc.addResponse("gettransaction", json{{"time", 82}, {"confirmations", 0}});
auto result = Refresh::collectTransactionRefreshResult(rpc, snap, 110, 4);
EXPECT_EQ(rpc.calls[0].method, std::string("z_viewtransaction"));
EXPECT_TRUE(result.transparentRowsReused);
EXPECT_EQ(countRows(result, "r1"), 1);
EXPECT_EQ(countRows(result, "s1"), 1);
EXPECT_EQ(countRows(result, "t-sent"), 1);
}
{ // The between-blocks poll without its probe: listtransactions only.
Refresh::TransactionRefreshSnapshot snap;
snap.shieldedAddresses = {"zs-a", "zs-b"};
snap.maxShieldedReceiveScans = 1;
snap.skipShieldedScans = true;
MockRefreshRpc rpc;
rpc.addResponse("listtransactions", json::array());
auto recent = Refresh::collectRecentTransactionRefreshResult(rpc, snap, 110);
EXPECT_TRUE(rpc.methodNames() == std::vector<std::string>({"listtransactions"}));
EXPECT_EQ(recent.shieldedAddressesScanned, static_cast<size_t>(0));
}
}
int main()
{
testConnectionConfig();
@@ -9863,6 +10499,11 @@ int main()
testParseMoneyZat();
testSpendableBalanceNeeded();
testOperationFilterParams();
testHistoryScanPolicy();
testShieldedHistoryPass();
testHistoryPassWithCollector();
testHistoryCollectorSweepAndEnrichment();
testHistoryDutyModel();
testNetworkRefreshResultModels();
testTrackedSendRetirement();
testBalanceDisplayLerp();