perf(lite-backend): share unchanged witness states between blocks
Each scanned block deep-copied the newest witness of every note with history to start that block's entry, although a block without shielded outputs changes no witness, and spent or pending notes never change theirs. A note's 100-entry list was 100 separate copies, mostly identical. The list now holds Arc-shared witnesses. The per-block duplicate is an Arc::clone, and Arc::make_mut copies the newest entry only for unspent, non-pending notes and only in blocks with at least one shielded output. Every entry is still written in full, so the file is unchanged, and a spend takes its own copy of the witness. A loaded wallet is not de-duplicated, so the saving builds up over the next 100 blocks. No output change: the saved bytes are identical; notes use less memory. With 1000 live notes x 100 witnesses (a 70 MB file), RssAnon after 100 blocks without outputs is 18 MiB instead of 189 MiB and such a block scans in 1.0 ms instead of 2.1-2.7 ms; when every block has an output, the live bytes stay about the same. The witness tests, the 1000-seed oracle and a differential run against the unmodified backend (every witness entry and the whole saved file after every operation; only the cleared lists and the chosen spend notes may differ) all pass, and the measured wallet saves to the same SHA-256 as with the old build. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+15
-9
@@ -2304,10 +2304,10 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64
|
|||||||
// Prepare the note witnesses for updating
|
// Prepare the note witnesses for updating
|
||||||
for tx in txs.values_mut() {
|
for tx in txs.values_mut() {
|
||||||
for nd in tx.notes.iter_mut() {
|
for nd in tx.notes.iter_mut() {
|
||||||
// Duplicate the most recent witness (nothing to do for notes with no history)
|
// Duplicate the most recent witness (shares the allocation; deep-cloned only when appended to)
|
||||||
if let Some(witness) = nd.witnesses.last() {
|
if let Some(witness) = nd.witnesses.last() {
|
||||||
let clone = witness.clone();
|
let shared = Arc::clone(witness);
|
||||||
nd.witnesses.push(clone);
|
nd.witnesses.push(shared);
|
||||||
// Trim the oldest witnesses in place (no fresh Vec per note per block)
|
// Trim the oldest witnesses in place (no fresh Vec per note per block)
|
||||||
let len = nd.witnesses.len();
|
let len = nd.witnesses.len();
|
||||||
if len > MAX_REORG {
|
if len > MAX_REORG {
|
||||||
@@ -2322,12 +2322,18 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64
|
|||||||
let extfvks: Vec<ExtendedFullViewingKey> = self.zkeys.read().unwrap().iter().map(|zk| zk.extfvk.clone()).collect();
|
let extfvks: Vec<ExtendedFullViewingKey> = self.zkeys.read().unwrap().iter().map(|zk| zk.extfvk.clone()).collect();
|
||||||
|
|
||||||
// Create a single mutable slice of all the newly-added witnesses.
|
// Create a single mutable slice of all the newly-added witnesses.
|
||||||
let mut witness_refs: Vec<_> = txs
|
// A block without shielded outputs appends nothing to any witness: don't un-share (deep-clone) them.
|
||||||
.values_mut()
|
let block_has_outputs = block.vtx.iter().any(|t| !t.outputs.is_empty());
|
||||||
.map(|tx| tx.notes.iter_mut().filter_map(
|
let mut witness_refs: Vec<&mut IncrementalWitness<Node>> = if block_has_outputs {
|
||||||
|nd| if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { nd.witnesses.last_mut() } else { None }))
|
txs
|
||||||
.flatten()
|
.values_mut()
|
||||||
.collect();
|
.map(|tx| tx.notes.iter_mut().filter_map(
|
||||||
|
|nd| if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { nd.witnesses.last_mut().map(|w| Arc::make_mut(w)) } else { None }))
|
||||||
|
.flatten()
|
||||||
|
.collect()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
|
||||||
self.scan_block_internal(
|
self.scan_block_internal(
|
||||||
block.clone(),
|
block.clone(),
|
||||||
|
|||||||
+6
-4
@@ -1,4 +1,5 @@
|
|||||||
use std::io::{self, Read, Write};
|
use std::io::{self, Read, Write};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
|
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
|
||||||
use pairing::bls12_381::{Bls12};
|
use pairing::bls12_381::{Bls12};
|
||||||
@@ -66,7 +67,8 @@ pub struct SaplingNoteData {
|
|||||||
pub(super) extfvk: ExtendedFullViewingKey, // Technically, this should be recoverable from the account number, but we're going to refactor this in the future, so I'll write it again here.
|
pub(super) extfvk: ExtendedFullViewingKey, // Technically, this should be recoverable from the account number, but we're going to refactor this in the future, so I'll write it again here.
|
||||||
pub diversifier: Diversifier,
|
pub diversifier: Diversifier,
|
||||||
pub note: Note<Bls12>,
|
pub note: Note<Bls12>,
|
||||||
pub(super) witnesses: Vec<IncrementalWitness<Node>>,
|
// One entry per block (<= MAX_REORG), oldest first. Entries of blocks that appended nothing share one allocation.
|
||||||
|
pub(super) witnesses: Vec<Arc<IncrementalWitness<Node>>>,
|
||||||
pub(super) nullifier: [u8; 32],
|
pub(super) nullifier: [u8; 32],
|
||||||
pub spent: Option<TxId>, // If this note was confirmed spent
|
pub spent: Option<TxId>, // If this note was confirmed spent
|
||||||
pub unconfirmed_spent: Option<TxId>, // If this note was spent in a send, but has not yet been confirmed.
|
pub unconfirmed_spent: Option<TxId>, // If this note was spent in a send, but has not yet been confirmed.
|
||||||
@@ -130,7 +132,7 @@ impl SaplingNoteData {
|
|||||||
extfvk: extfvk.clone(),
|
extfvk: extfvk.clone(),
|
||||||
diversifier: output.to.diversifier,
|
diversifier: output.to.diversifier,
|
||||||
note: output.note,
|
note: output.note,
|
||||||
witnesses: vec![witness],
|
witnesses: vec![Arc::new(witness)],
|
||||||
nullifier: nf,
|
nullifier: nf,
|
||||||
spent: None,
|
spent: None,
|
||||||
unconfirmed_spent: None,
|
unconfirmed_spent: None,
|
||||||
@@ -162,7 +164,7 @@ impl SaplingNoteData {
|
|||||||
None => Err(io::Error::new(io::ErrorKind::InvalidInput, "Couldn't create the note for the address"))
|
None => Err(io::Error::new(io::ErrorKind::InvalidInput, "Couldn't create the note for the address"))
|
||||||
}?;
|
}?;
|
||||||
|
|
||||||
let witnesses = Vector::read(&mut reader, |r| IncrementalWitness::<Node>::read(r))?;
|
let witnesses = Vector::read(&mut reader, |r| IncrementalWitness::<Node>::read(r).map(Arc::new))?;
|
||||||
|
|
||||||
let mut nullifier = [0u8; 32];
|
let mut nullifier = [0u8; 32];
|
||||||
reader.read_exact(&mut nullifier)?;
|
reader.read_exact(&mut nullifier)?;
|
||||||
@@ -553,7 +555,7 @@ impl SpendableNote {
|
|||||||
nullifier: nd.nullifier,
|
nullifier: nd.nullifier,
|
||||||
diversifier: nd.diversifier,
|
diversifier: nd.diversifier,
|
||||||
note: nd.note.clone(),
|
note: nd.note.clone(),
|
||||||
witness: w.clone(),
|
witness: (**w).clone(),
|
||||||
extsk: extsk.clone().unwrap(),
|
extsk: extsk.clone().unwrap(),
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
Reference in New Issue
Block a user