wallet: cache each Sapling note's decrypted plaintext in memory
GetFilteredNotes, the walk behind every balance and note-listing RPC,
sweep, consolidation, z_sendmany and z_mergetoaddress, decrypted every
note it looked at on every call: ~0.5ms of Jubjub arithmetic per note,
under LOCK2(cs_main, cs_wallet). A note's plaintext is fixed by its
output (the txid commits to the ciphertext) and the ivk that decrypts
it, so it never needs to be decrypted twice.
SaplingNoteData gains an in-memory-only
boost::optional<SaplingNoteCache> {note, memo}; the address is the
note's (d, pk_d). It follows the witnessRootValidated precedent:
- never serialized, and left out of operator== and
SaplingNoteDataIdentical, so filling it never makes a tx look changed
and get rewritten (the replay rewrite storm 69bb035e7 removed);
- FindMySaplingNotes fills it from the decryption that found the note
(both the spending-key and the viewing-key-only loop);
- UpdatedNoteData's replace branch keeps a plaintext the wallet already
had, for the same outpoint and ivk only;
- GetFilteredNotes now runs in two passes: pass 1 applies the tx,
spent and locked filters and collects the notes that pass; whatever
lacks a plaintext (everything after a restart, until the startup
warm-up in a later commit) is decrypted once via the new
FillSaplingNoteCache; pass 2 applies the address and spending-key
filters and emits entries in the same order as before.
Spentness, spendability, lock state and depth are still decided on every
call; only the plaintext is cached.
The A0 equivalence test passes unchanged (warm). New tests:
- ColdCacheMatchesLegacy: the same grid with the cache dropped before
every call;
- CachedPlaintextMatchesFreshDecrypt: eager and lazy fills;
- WarmPassDoesNotDecrypt: a fill counter shows a warm walk decrypts
nothing, and spent or locked notes only when the caller asks for
them;
- CacheNeverSerialized: byte-identical SER_DISK/NETWORK/GETHASH output,
empty after a load;
- UpdatedNoteDataAndTheCache: no rewrite on a cache-only difference,
carried on replace, never across another ivk;
- SpendabilityNotCached;
- FindMySaplingNotesFillsCache.
Memory: ~616 bytes per note (the memo is 512), ~8MB for a 13k-note
wallet, next to ~100KB per note of cached witnesses.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+77
-35
@@ -2078,6 +2078,14 @@ bool CWallet::UpdatedNoteData(const CWalletTx& wtxIn, CWalletTx& wtx, bool fSame
|
||||
}
|
||||
}
|
||||
|
||||
// Keep a decrypted plaintext the wallet already holds (in memory only; a pure function of the
|
||||
// outpoint and ivk) when the incoming note data lacks one.
|
||||
for (const mapSaplingNoteData_t::value_type& nd : wtx.mapSaplingNoteData) {
|
||||
mapSaplingNoteData_t::iterator it = tmp.find(nd.first);
|
||||
if (it != tmp.end() && !it->second.cachedPlaintext && nd.second.cachedPlaintext && it->second.ivk == nd.second.ivk)
|
||||
it->second.cachedPlaintext = nd.second.cachedPlaintext;
|
||||
}
|
||||
|
||||
// A replayed or rescanned tx usually carries nothing new; don't rewrite it to disk. Do have
|
||||
// VerifyAndSetInitialWitness re-check the kept witnesses against the active chain once (the
|
||||
// flag is in memory only): the chain may have moved under them, and it keeps a rescan able to
|
||||
@@ -2280,6 +2288,7 @@ std::pair<mapSaplingNoteData_t, SaplingIncomingViewingKeyMap> CWallet::FindMySap
|
||||
SaplingOutPoint op {hash, i};
|
||||
SaplingNoteData nd;
|
||||
nd.ivk = ivk;
|
||||
nd.cachedPlaintext = SaplingNoteCache{ result->second, result->first.memo() };
|
||||
noteData.insert(std::make_pair(op, nd));
|
||||
found = true;
|
||||
break;
|
||||
@@ -2288,7 +2297,7 @@ std::pair<mapSaplingNoteData_t, SaplingIncomingViewingKeyMap> CWallet::FindMySap
|
||||
if (!found) {
|
||||
for (auto it = mapSaplingIncomingViewingKeys.begin(); it != mapSaplingIncomingViewingKeys.end(); ++it) {
|
||||
SaplingIncomingViewingKey ivk = it-> second;
|
||||
auto result = SaplingNotePlaintext::decrypt(output.encCiphertext, ivk, output.ephemeralKey, output.cm);
|
||||
auto result = SaplingNotePlaintext::decrypt_with_note(output.encCiphertext, ivk, output.ephemeralKey, output.cm);
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
@@ -2297,6 +2306,7 @@ std::pair<mapSaplingNoteData_t, SaplingIncomingViewingKeyMap> CWallet::FindMySap
|
||||
SaplingOutPoint op {hash, i};
|
||||
SaplingNoteData nd;
|
||||
nd.ivk = ivk;
|
||||
nd.cachedPlaintext = SaplingNoteCache{ result->second, result->first.memo() };
|
||||
noteData.insert(std::make_pair(op, nd));
|
||||
break;
|
||||
}
|
||||
@@ -5717,6 +5727,22 @@ bool CMerkleTx::AcceptToMemoryPool(bool fLimitFree, bool fRejectAbsurdFee)
|
||||
return ::AcceptToMemoryPool(mempool, state, *this, fLimitFree, NULL, fRejectAbsurdFee);
|
||||
}
|
||||
|
||||
unsigned int CWallet::FillSaplingNoteCache(const std::vector<SaplingNoteToDecrypt>& notes)
|
||||
{
|
||||
AssertLockHeld(cs_wallet);
|
||||
unsigned int nFilled = 0;
|
||||
for (const SaplingNoteToDecrypt& n : notes) {
|
||||
auto result = SaplingNotePlaintext::decrypt_with_note(
|
||||
n.output->encCiphertext, n.nd->ivk, n.output->ephemeralKey, n.output->cm);
|
||||
if (result) {
|
||||
n.nd->cachedPlaintext = SaplingNoteCache{ result->second, result->first.memo() };
|
||||
nFilled++;
|
||||
}
|
||||
}
|
||||
nSaplingNoteCacheFills += nFilled;
|
||||
return nFilled;
|
||||
}
|
||||
|
||||
// Replaces tx_height() on the wallet's note paths. tx_height() re-read the tx from disk through
|
||||
// GetTransaction, once per wallet tx (or per returned note) under LOCK2, and on a -txindex=0 node it
|
||||
// never found a fully shielded tx, so those notes were silently skipped. The depth already locates
|
||||
@@ -5769,10 +5795,19 @@ void CWallet::GetFilteredNotes(
|
||||
const size_t nEntriesBefore = saplingEntries.size();
|
||||
unsigned int nTxs = 0, nNotes = 0, nSpent = 0;
|
||||
|
||||
// Pass 1: the notes that pass every filter not needing the plaintext, in wallet order.
|
||||
struct Candidate {
|
||||
const SaplingOutPoint* op;
|
||||
const SaplingNoteData* nd;
|
||||
int nDepth;
|
||||
};
|
||||
std::vector<Candidate> candidates;
|
||||
std::vector<SaplingNoteToDecrypt> uncached;
|
||||
for (auto & p : mapWallet) {
|
||||
// By reference: a copy deep-copies every note's witness list (up to WITNESS_CACHE_SIZE
|
||||
// witnesses each), and it was made before any filter ran.
|
||||
const CWalletTx& wtx = p.second;
|
||||
// witnesses each), and it was made before any filter ran. Not const: a missing plaintext is
|
||||
// cached onto the wallet's own entry.
|
||||
CWalletTx& wtx = p.second;
|
||||
|
||||
// Most wallet txs (coinbase, sends from other wallets) carry no notes of ours.
|
||||
if (wtx.mapSaplingNoteData.empty())
|
||||
@@ -5800,13 +5835,13 @@ void CWallet::GetFilteredNotes(
|
||||
}
|
||||
nTxs++;
|
||||
|
||||
for (const auto & pair : wtx.mapSaplingNoteData) {
|
||||
for (auto & pair : wtx.mapSaplingNoteData) {
|
||||
const SaplingOutPoint& op = pair.first;
|
||||
const SaplingNoteData& nd = pair.second;
|
||||
SaplingNoteData& nd = pair.second;
|
||||
nNotes++;
|
||||
|
||||
// The cheap filters run before the decryption, which costs ~0.5-0.8ms of Jubjub
|
||||
// arithmetic per note. Spent notes stay in the wallet (-deletetx is off by default).
|
||||
// The cheap filters run before any decryption, which costs ~0.5ms of Jubjub arithmetic
|
||||
// per note. Spent notes stay in the wallet (-deletetx is off by default).
|
||||
if (ignoreSpent && nd.nullifier && IsSaplingSpent(*nd.nullifier)) {
|
||||
nSpent++;
|
||||
continue;
|
||||
@@ -5818,36 +5853,43 @@ void CWallet::GetFilteredNotes(
|
||||
continue;
|
||||
}
|
||||
|
||||
// One key agreement and one pk_d derivation; the note and its address come with it.
|
||||
auto maybe_pt = SaplingNotePlaintext::decrypt_with_note(
|
||||
wtx.vShieldedOutput[op.n].encCiphertext,
|
||||
nd.ivk,
|
||||
wtx.vShieldedOutput[op.n].ephemeralKey,
|
||||
wtx.vShieldedOutput[op.n].cm);
|
||||
assert(static_cast<bool>(maybe_pt));
|
||||
const SaplingNotePlaintext& notePt = maybe_pt->first;
|
||||
const libzcash::SaplingNote& note = maybe_pt->second;
|
||||
const libzcash::SaplingPaymentAddress pa(note.d, note.pk_d);
|
||||
assert(op.n < wtx.vShieldedOutput.size());
|
||||
candidates.push_back(Candidate { &op, &nd, nDepth });
|
||||
if (!nd.cachedPlaintext)
|
||||
uncached.push_back(SaplingNoteToDecrypt { &nd, &wtx.vShieldedOutput[op.n] });
|
||||
}
|
||||
}
|
||||
|
||||
// skip notes which belong to a different payment address in the wallet
|
||||
if (!(filterAddresses.empty() || filterAddresses.count(pa))) {
|
||||
// Decrypt what the cache lacks: normally nothing, since FindMySaplingNotes fills new notes and the
|
||||
// startup warm-up fills the ones loaded from wallet.dat.
|
||||
const unsigned int nFilled = uncached.empty() ? 0 : FillSaplingNoteCache(uncached);
|
||||
|
||||
// Pass 2: the filters that need the note's address.
|
||||
saplingEntries.reserve(saplingEntries.size() + candidates.size());
|
||||
for (const Candidate& c : candidates) {
|
||||
// A wallet note that doesn't decrypt under its own ivk is corrupt; as before, that's fatal here.
|
||||
assert(static_cast<bool>(c.nd->cachedPlaintext));
|
||||
const SaplingNoteCache& plaintext = *c.nd->cachedPlaintext;
|
||||
const libzcash::SaplingPaymentAddress pa(plaintext.note.d, plaintext.note.pk_d);
|
||||
|
||||
// skip notes which belong to a different payment address in the wallet
|
||||
if (!(filterAddresses.empty() || filterAddresses.count(pa))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// skip notes which cannot be spent
|
||||
if (requireSpendingKey) {
|
||||
libzcash::SaplingIncomingViewingKey ivk;
|
||||
libzcash::SaplingFullViewingKey fvk;
|
||||
if (!(GetSaplingIncomingViewingKey(pa, ivk) &&
|
||||
GetSaplingFullViewingKey(ivk, fvk) &&
|
||||
HaveSaplingSpendingKey(fvk))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// skip notes which cannot be spent
|
||||
if (requireSpendingKey) {
|
||||
libzcash::SaplingIncomingViewingKey ivk;
|
||||
libzcash::SaplingFullViewingKey fvk;
|
||||
if (!(GetSaplingIncomingViewingKey(pa, ivk) &&
|
||||
GetSaplingFullViewingKey(ivk, fvk) &&
|
||||
HaveSaplingSpendingKey(fvk))) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
saplingEntries.push_back(SaplingNoteEntry {
|
||||
op, pa, note, notePt.memo(), nDepth });
|
||||
}
|
||||
|
||||
saplingEntries.push_back(SaplingNoteEntry {
|
||||
*c.op, pa, plaintext.note, plaintext.memo, c.nDepth });
|
||||
}
|
||||
|
||||
// A slow walk holds cs_main long enough to stall block connection, so it is always logged; a
|
||||
@@ -5855,9 +5897,9 @@ void CWallet::GetFilteredNotes(
|
||||
// counts go to the default log only with -debug=bench too.
|
||||
const int64_t nWalkMs = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - walkStart).count();
|
||||
if (LogAcceptCategory("bench")) {
|
||||
LogPrintf("%s: minDepth=%d ignoreSpent=%d requireSpendingKey=%d ignoreLocked=%d txs=%u/%u notes=%u spent=%u entries=%u walk=%dms\n",
|
||||
LogPrintf("%s: minDepth=%d ignoreSpent=%d requireSpendingKey=%d ignoreLocked=%d txs=%u/%u notes=%u spent=%u decrypted=%u entries=%u walk=%dms\n",
|
||||
__func__, minDepth, ignoreSpent, requireSpendingKey, ignoreLocked, nTxs, (unsigned int)mapWallet.size(),
|
||||
nNotes, nSpent, (unsigned int)(saplingEntries.size() - nEntriesBefore), nWalkMs);
|
||||
nNotes, nSpent, nFilled, (unsigned int)(saplingEntries.size() - nEntriesBefore), nWalkMs);
|
||||
} else if (nWalkMs >= 500) {
|
||||
LogPrintf("%s: minDepth=%d ignoreSpent=%d requireSpendingKey=%d ignoreLocked=%d walk=%dms\n",
|
||||
__func__, minDepth, ignoreSpent, requireSpendingKey, ignoreLocked, nWalkMs);
|
||||
|
||||
Reference in New Issue
Block a user