From ba08cab95a8f8a6d9404769d2292f812bbc91762 Mon Sep 17 00:00:00 2001 From: DanS Date: Sat, 10 Oct 2026 09:50:03 -0500 Subject: [PATCH] perf(lite-backend): share unchanged witness states between blocks Each scanned block deep-copied the newest witness of every note with history to start that block's entry, although a block without shielded outputs changes no witness, and spent or pending notes never change theirs. A note's 100-entry list was 100 separate copies, mostly identical. The list now holds Arc-shared witnesses. The per-block duplicate is an Arc::clone, and Arc::make_mut copies the newest entry only for unspent, non-pending notes and only in blocks with at least one shielded output. Every entry is still written in full, so the file is unchanged, and a spend takes its own copy of the witness. A loaded wallet is not de-duplicated, so the saving builds up over the next 100 blocks. No output change: the saved bytes are identical; notes use less memory. With 1000 live notes x 100 witnesses (a 70 MB file), RssAnon after 100 blocks without outputs is 18 MiB instead of 189 MiB and such a block scans in 1.0 ms instead of 2.1-2.7 ms; when every block has an output, the live bytes stay about the same. The witness tests, the 1000-seed oracle and a differential run against the unmodified backend (every witness entry and the whole saved file after every operation; only the cleared lists and the chosen spend notes may differ) all pass, and the measured wallet saves to the same SHA-256 as with the old build. Co-Authored-By: Claude Opus 5.5 --- .../lib/src/lightwallet.rs | 24 ++++++++++++------- .../lib/src/lightwallet/data.rs | 10 ++++---- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs index 371a70a..50bb3c5 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs @@ -2304,10 +2304,10 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 // Prepare the note witnesses for updating for tx in txs.values_mut() { for nd in tx.notes.iter_mut() { - // Duplicate the most recent witness (nothing to do for notes with no history) + // Duplicate the most recent witness (shares the allocation; deep-cloned only when appended to) if let Some(witness) = nd.witnesses.last() { - let clone = witness.clone(); - nd.witnesses.push(clone); + let shared = Arc::clone(witness); + nd.witnesses.push(shared); // Trim the oldest witnesses in place (no fresh Vec per note per block) let len = nd.witnesses.len(); if len > MAX_REORG { @@ -2322,12 +2322,18 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 let extfvks: Vec = self.zkeys.read().unwrap().iter().map(|zk| zk.extfvk.clone()).collect(); // Create a single mutable slice of all the newly-added witnesses. - let mut witness_refs: Vec<_> = txs - .values_mut() - .map(|tx| tx.notes.iter_mut().filter_map( - |nd| if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { nd.witnesses.last_mut() } else { None })) - .flatten() - .collect(); + // A block without shielded outputs appends nothing to any witness: don't un-share (deep-clone) them. + let block_has_outputs = block.vtx.iter().any(|t| !t.outputs.is_empty()); + let mut witness_refs: Vec<&mut IncrementalWitness> = if block_has_outputs { + txs + .values_mut() + .map(|tx| tx.notes.iter_mut().filter_map( + |nd| if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { nd.witnesses.last_mut().map(|w| Arc::make_mut(w)) } else { None })) + .flatten() + .collect() + } else { + Vec::new() + }; self.scan_block_internal( block.clone(), diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/data.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/data.rs index 3db0d1a..72157ed 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/data.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/data.rs @@ -1,4 +1,5 @@ use std::io::{self, Read, Write}; +use std::sync::Arc; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use pairing::bls12_381::{Bls12}; @@ -66,7 +67,8 @@ pub struct SaplingNoteData { pub(super) extfvk: ExtendedFullViewingKey, // Technically, this should be recoverable from the account number, but we're going to refactor this in the future, so I'll write it again here. pub diversifier: Diversifier, pub note: Note, - pub(super) witnesses: Vec>, + // One entry per block (<= MAX_REORG), oldest first. Entries of blocks that appended nothing share one allocation. + pub(super) witnesses: Vec>>, pub(super) nullifier: [u8; 32], pub spent: Option, // If this note was confirmed spent pub unconfirmed_spent: Option, // If this note was spent in a send, but has not yet been confirmed. @@ -130,7 +132,7 @@ impl SaplingNoteData { extfvk: extfvk.clone(), diversifier: output.to.diversifier, note: output.note, - witnesses: vec![witness], + witnesses: vec![Arc::new(witness)], nullifier: nf, spent: None, unconfirmed_spent: None, @@ -162,7 +164,7 @@ impl SaplingNoteData { None => Err(io::Error::new(io::ErrorKind::InvalidInput, "Couldn't create the note for the address")) }?; - let witnesses = Vector::read(&mut reader, |r| IncrementalWitness::::read(r))?; + let witnesses = Vector::read(&mut reader, |r| IncrementalWitness::::read(r).map(Arc::new))?; let mut nullifier = [0u8; 32]; reader.read_exact(&mut nullifier)?; @@ -553,7 +555,7 @@ impl SpendableNote { nullifier: nd.nullifier, diversifier: nd.diversifier, note: nd.note.clone(), - witness: w.clone(), + witness: (**w).clone(), extsk: extsk.clone().unwrap(), }) } else {