wallet: cache each Sapling note's decrypted plaintext in memory

GetFilteredNotes, the walk behind every balance and note-listing RPC, sweep, consolidation, z_sendmany and z_mergetoaddress, decrypted every note it looked at on every call: ~0.5ms of Jubjub arithmetic per note, under LOCK2(cs_main, cs_wallet). A note's plaintext is fixed by its output (the txid commits to the ciphertext) and the ivk that decrypts it, so it never needs to be decrypted twice. SaplingNoteData gains an in-memory-only boost::optional<SaplingNoteCache> {note, memo}; the address is the note's (d, pk_d). It follows the witnessRootValidated precedent: - never serialized, and left out of operator== and SaplingNoteDataIdentical, so filling it never makes a tx look changed and get rewritten (the replay rewrite storm 69bb035e7 removed); - FindMySaplingNotes fills it from the decryption that found the note (both the spending-key and the viewing-key-only loop); - UpdatedNoteData's replace branch keeps a plaintext the wallet already had, for the same outpoint and ivk only; - GetFilteredNotes now runs in two passes: pass 1 applies the tx, spent and locked filters and collects the notes that pass; whatever lacks a plaintext (everything after a restart, until the startup warm-up in a later commit) is decrypted once via the new FillSaplingNoteCache; pass 2 applies the address and spending-key filters and emits entries in the same order as before. Spentness, spendability, lock state and depth are still decided on every call; only the plaintext is cached. The A0 equivalence test passes unchanged (warm). New tests: - ColdCacheMatchesLegacy: the same grid with the cache dropped before every call; - CachedPlaintextMatchesFreshDecrypt: eager and lazy fills; - WarmPassDoesNotDecrypt: a fill counter shows a warm walk decrypts nothing, and spent or locked notes only when the caller asks for them; - CacheNeverSerialized: byte-identical SER_DISK/NETWORK/GETHASH output, empty after a load; - UpdatedNoteDataAndTheCache: no rewrite on a cache-only difference, carried on replace, never across another ivk; - SpendabilityNotCached; - FindMySaplingNotesFillsCache. Memory: ~616 bytes per note (the memo is 512), ~8MB for a 13k-note wallet, next to ~100KB per note of cached witnesses. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 08:16:09 -05:00
parent 7ddd03d86d
commit b1e3c5230a
3 changed files with 289 additions and 36 deletions
@@ -121,8 +121,38 @@ class NoteCacheWallet : public CWallet {
 public:
    bool EncryptKeys(CKeyingMaterial& vMasterKeyIn) { return CCryptoKeyStore::EncryptKeys(vMasterKeyIn); }
    bool Unlock(const CKeyingMaterial& vMasterKeyIn) { return CCryptoKeyStore::Unlock(vMasterKeyIn); }
+    bool UpdatedNoteData(const CWalletTx& wtxIn, CWalletTx& wtx, bool fSameBlock = true) {
+        return CWallet::UpdatedNoteData(wtxIn, wtx, fSameBlock);
+    }
+
+    // Drops every note's in-memory plaintext, as a restart does.
+    void ClearPlaintextCache() {
+        for (auto& p : mapWallet)
+            for (auto& item : p.second.mapSaplingNoteData)
+                item.second.cachedPlaintext = boost::none;
+    }
+    size_t CachedNotes() const {
+        size_t n = 0;
+        for (const auto& p : mapWallet)
+            for (const auto& item : p.second.mapSaplingNoteData)
+                n += item.second.cachedPlaintext ? 1 : 0;
+        return n;
+    }
 };

+void ExpectCacheMatchesFreshDecrypt(const SaplingNoteData& nd, const OutputDescription& od)
+{
+    ASSERT_TRUE(static_cast<bool>(nd.cachedPlaintext));
+    auto fresh = SaplingNotePlaintext::decrypt_with_note(od.encCiphertext, nd.ivk, od.ephemeralKey, od.cm);
+    ASSERT_TRUE(static_cast<bool>(fresh));
+    const SaplingNote& cached = nd.cachedPlaintext->note;
+    EXPECT_TRUE(cached.d == fresh->second.d);
+    EXPECT_EQ(fresh->second.pk_d, cached.pk_d);
+    EXPECT_EQ(fresh->second.r, cached.r);
+    EXPECT_EQ(fresh->second.value(), cached.value());
+    EXPECT_TRUE(nd.cachedPlaintext->memo == fresh->first.memo());
+}
+
 // The fake active chain is 0..TIP_HEIGHT; STALE_HEIGHT is a side block forked off it.
 constexpr int TIP_HEIGHT = 30;
 constexpr int STALE_HEIGHT = 25;
@@ -357,7 +387,8 @@ protected:
        return entries;
    }

-    void ExpectEquivalent(const std::vector<int>& minDepths) {
+    // cold: drop the plaintext cache before every call, so each one decrypts every note it returns.
+    void ExpectEquivalent(const std::vector<int>& minDepths, bool cold = false) {
        const std::vector<std::pair<std::string, std::set<PaymentAddress>>> filters = {
            {"none", {}}, {"A", {paA}}, {"B", {paB}}, {"A+C", {paA, paC}},
        };
@@ -368,6 +399,10 @@ protected:
        for (bool ignoreLocked : {false, true})
        for (const auto& filter : filters) {
            std::string ctx = Describe(minDepth, maxDepth, ignoreSpent, requireSpendingKey, ignoreLocked, filter.first);
+            if (cold) {
+                LOCK(wallet.cs_wallet);
+                wallet.ClearPlaintextCache();
+            }
            ExpectSameEntries(Legacy(filter.second, minDepth, maxDepth, ignoreSpent, requireSpendingKey, ignoreLocked),
                              Filtered(filter.second, minDepth, maxDepth, ignoreSpent, requireSpendingKey, ignoreLocked),
                              ctx);
@@ -588,3 +623,150 @@ TEST(WalletNoteCacheDiscovery, FindMySaplingNotes) {
    EXPECT_TRUE(toAdd.at(addrsA[1]) == ivkA);
    EXPECT_TRUE(toAdd.at(addrsA[2]) == ivkA);
 }
+
+// Every note FindMySaplingNotes finds comes with its plaintext, for both the spending-key loop and the
+// viewing-key-only loop.
+TEST(WalletNoteCacheDiscovery, FindMySaplingNotesFillsCache) {
+    NoteCacheWallet wallet;
+    auto skA = TestKey(0), skB = TestKey(1);
+    auto addrsA = AddressesOf(skA);
+    LOCK2(cs_main, wallet.cs_wallet);
+    ASSERT_TRUE(wallet.AddSaplingZKey(skA, addrsA[0]));
+    ASSERT_TRUE(wallet.AddSaplingIncomingViewingKey(skB.expsk.full_viewing_key().in_viewing_key(), skB.DefaultAddress()));
+    CTransaction tx = MakeTx({MakeOutput(addrsA[0], 11), MakeOutput(addrsA[1], 12), MakeOutput(skB.DefaultAddress(), 13)});
+    auto notes = wallet.FindMySaplingNotes(tx).first;
+    ASSERT_EQ(3u, notes.size());
+    for (const auto& item : notes)
+        ExpectCacheMatchesFreshDecrypt(item.second, tx.vShieldedOutput[item.first.n]);
+}
+
+// With the cache dropped before every call (a restart), the walk still returns exactly the old loop's
+// entries: the lazy fill decrypts the same notes the same way.
+TEST_F(WalletNoteCache, ColdCacheMatchesLegacy) {
+    ExpectEquivalent({0, 1, 11}, true);
+}
+
+// The cache holds exactly what a fresh decryption gives, wherever it was filled.
+TEST_F(WalletNoteCache, CachedPlaintextMatchesFreshDecrypt) {
+    LOCK2(cs_main, wallet.cs_wallet);
+    for (int pass = 0; pass < 2; pass++) {
+        if (pass == 1) {
+            wallet.ClearPlaintextCache();
+            Filtered({}, 0, INT_MAX, false, false, false);   // lazy fill of every confirmed note
+        }
+        size_t checked = 0;
+        for (const auto& p : wallet.mapWallet) {
+            for (const auto& item : p.second.mapSaplingNoteData) {
+                if (!item.second.cachedPlaintext)
+                    continue;
+                ExpectCacheMatchesFreshDecrypt(item.second, p.second.vShieldedOutput[item.first.n]);
+                checked++;
+            }
+        }
+        EXPECT_GE(checked, 9u) << "pass " << pass;
+    }
+}
+
+// A warm walk decrypts nothing; a cold one decrypts each note it needs exactly once, and spent notes
+// only when the caller asks for them.
+TEST_F(WalletNoteCache, WarmPassDoesNotDecrypt) {
+    LOCK2(cs_main, wallet.cs_wallet);
+    const uint64_t base = wallet.nSaplingNoteCacheFills;
+    Filtered({}, 1, INT_MAX, true, true, true);
+    EXPECT_EQ(base, wallet.nSaplingNoteCacheFills) << "FindMySaplingNotes already filled every note";
+
+    wallet.ClearPlaintextCache();
+    EXPECT_EQ(0u, wallet.CachedNotes());
+    Filtered({}, 1, INT_MAX, true, false, true);
+    // tx1 (2), tx3 (3), tx4 (1), tx7 (1): tx2's note is locked and tx6's is spent, so neither is decrypted.
+    EXPECT_EQ(base + 7, wallet.nSaplingNoteCacheFills);
+    Filtered({}, 1, INT_MAX, true, false, true);
+    Filtered({paA}, 0, INT_MAX, true, true, true);
+    EXPECT_EQ(base + 7, wallet.nSaplingNoteCacheFills) << "warm";
+
+    Filtered({}, 1, INT_MAX, false, false, false);
+    EXPECT_EQ(base + 9, wallet.nSaplingNoteCacheFills) << "the locked and the spent note, once";
+    Filtered({}, 1, INT_MAX, false, false, false);
+    EXPECT_EQ(base + 9, wallet.nSaplingNoteCacheFills);
+}
+
+// The cache never reaches wallet.dat, the network or a tx hash: serialization is byte-identical with it
+// filled or empty, and a deserialized entry comes back empty.
+TEST_F(WalletNoteCache, CacheNeverSerialized) {
+    LOCK2(cs_main, wallet.cs_wallet);
+    const uint256 txid = txids[2];   // tx3: three notes
+    CWalletTx filled = wallet.mapWallet.at(txid);
+    ASSERT_EQ(3u, filled.mapSaplingNoteData.size());
+    CWalletTx empty = filled;
+    for (auto& item : empty.mapSaplingNoteData) {
+        ASSERT_TRUE(static_cast<bool>(item.second.cachedPlaintext));
+        item.second.cachedPlaintext = boost::none;
+    }
+    for (int type : {SER_DISK, SER_NETWORK, SER_GETHASH}) {
+        CDataStream a(type, CLIENT_VERSION), b(type, CLIENT_VERSION);
+        a << filled;
+        b << empty;
+        EXPECT_TRUE(a.str() == b.str()) << "CWalletTx, type " << type;
+        CDataStream c(type, CLIENT_VERSION), d(type, CLIENT_VERSION);
+        c << filled.mapSaplingNoteData.begin()->second;
+        d << empty.mapSaplingNoteData.begin()->second;
+        EXPECT_TRUE(c.str() == d.str()) << "SaplingNoteData, type " << type;
+    }
+    EXPECT_EQ(txid, filled.GetHash());
+
+    CDataStream ss(SER_DISK, CLIENT_VERSION);
+    ss << filled;
+    CWalletTx back;
+    ss >> back;
+    ASSERT_EQ(3u, back.mapSaplingNoteData.size());
+    for (const auto& item : back.mapSaplingNoteData)
+        EXPECT_FALSE(static_cast<bool>(item.second.cachedPlaintext));
+}
+
+// UpdatedNoteData: a difference in the cache alone is no update (no rewrite to disk); the replace branch
+// keeps a plaintext the wallet already had; and nothing is carried across a different ivk.
+TEST_F(WalletNoteCache, UpdatedNoteDataAndTheCache) {
+    LOCK2(cs_main, wallet.cs_wallet);
+    const uint256 txid = txids[2];
+    CWalletTx& wtx = wallet.mapWallet.at(txid);
+    const SaplingOutPoint op0(txid, 0), op1(txid, 1);
+
+    // Same note data, but arriving without plaintext: unchanged, and the wallet keeps its cache.
+    CWalletTx replay = wtx;
+    for (auto& item : replay.mapSaplingNoteData)
+        item.second.cachedPlaintext = boost::none;
+    EXPECT_FALSE(wallet.UpdatedNoteData(replay, wtx));
+    EXPECT_TRUE(static_cast<bool>(wtx.mapSaplingNoteData.at(op0).cachedPlaintext));
+
+    // A real update (an extra note) without plaintext for the existing ones: replaced, cache carried.
+    CWalletTx incoming = replay;
+    incoming.mapSaplingNoteData.erase(op1);
+    wtx.mapSaplingNoteData.erase(op1);
+    incoming.mapSaplingNoteData[op1] = SaplingNoteData(TestKey(2).expsk.full_viewing_key().in_viewing_key());
+    EXPECT_TRUE(wallet.UpdatedNoteData(incoming, wtx));
+    ExpectCacheMatchesFreshDecrypt(wtx.mapSaplingNoteData.at(op0), wtx.vShieldedOutput[0]);
+    EXPECT_FALSE(static_cast<bool>(wtx.mapSaplingNoteData.at(op1).cachedPlaintext)) << "nothing to carry";
+
+    // Incoming note data under another ivk: its (absent) plaintext is not replaced by ours.
+    CWalletTx otherKey = wtx;
+    for (auto& item : otherKey.mapSaplingNoteData) {
+        item.second.ivk = TestKey(9).expsk.full_viewing_key().in_viewing_key();
+        item.second.cachedPlaintext = boost::none;
+    }
+    EXPECT_TRUE(wallet.UpdatedNoteData(otherKey, wtx));
+    for (const auto& item : wtx.mapSaplingNoteData)
+        EXPECT_FALSE(static_cast<bool>(item.second.cachedPlaintext));
+}
+
+// Spendability is decided per call, never cached: a note to a viewing-key-only address joins the
+// requireSpendingKey results as soon as its spending key is added, with its plaintext already cached.
+TEST_F(WalletNoteCache, SpendabilityNotCached) {
+    LOCK2(cs_main, wallet.cs_wallet);
+    auto before = Filtered({paB}, 1, INT_MAX, true, true, false);
+    EXPECT_EQ(0u, before.size());
+    EXPECT_EQ(2u, Filtered({paB}, 1, INT_MAX, true, false, false).size());
+    const uint64_t fills = wallet.nSaplingNoteCacheFills;
+    ASSERT_TRUE(wallet.AddSaplingZKey(TestKey(1), paB));
+    EXPECT_EQ(2u, Filtered({paB}, 1, INT_MAX, true, true, false).size());
+    EXPECT_EQ(fills, wallet.nSaplingNoteCacheFills);
+}