diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs index d18b2fd..d605c9e 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs @@ -2598,3 +2598,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 #[cfg(test)] pub mod tests; + +#[cfg(test)] +mod oracle_tests; diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs new file mode 100644 index 0000000..cf3062d --- /dev/null +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs @@ -0,0 +1,1784 @@ +// Copyright The Hush Developers 2019-2022 +// Released under the GPLv3 + +//! Seeded chain/witness oracle and offline spend verifier. +//! +//! A seeded generator builds random fake compact chains and feeds them to a `LightWallet` +//! through its public entry points (`scan_block_with_pool`, `invalidate_block`, `read`/`write`, +//! `send_to_address`, `scan_full_tx`). It covers receives to HD, diversified, imported-sk, +//! view-only and foreign keys, zero-value and dust notes, outputs with an unparseable epk, +//! compact spends with change, real `send_to_address` sends that are mined later (or never), +//! reorgs of 1..=100 blocks (in one call or block by block, with spends re-mined on the new +//! branch), and save/reload. +//! +//! An independent reference keeps its own `CommitmentTree`, the tree and root of every block, +//! and every note's true witness at each height. After every operation the wallet must agree: +//! - `get_sapling_tree()`, the retained block window and every retained block's tree; +//! - the wallet txs and, per note: txid, block, value, account, diversifier, nullifier, +//! spent/pending state, `is_change` and the tx's `total_shielded_value_spent`; +//! - witness-list lengths follow the legacy rule (one entry per block, at most 100, minus k on a +//! k-block invalidate). A spent note may instead hold an empty list once its spender is below +//! a full block window (the history can no longer be needed); +//! - every retained witness of an unspent, not-pending note equals (bytes) the reference witness +//! at that height, and sampled ones are checked semantically: root == reference root of that +//! block, position == the leaf index, and the authentication path hashes the note's cmu up to +//! the root; +//! - zbalance/verified/spendable balances (all, and per address at reload and at the end), the +//! `scan_block` and `invalidate_block` return values, and the mempool (including expiry). +//! +//! Spends are verified offline: the raw tx is captured in the broadcast closure (nothing is ever +//! sent anywhere). Every anchor must equal the reference root at tip - anchor_offset, every +//! nullifier must belong to a note that is eligible at that anchor, and the zcash_proofs +//! `SaplingVerificationContext` must accept every spend and output proof and the binding +//! signature. The first send of a run also checks that the verifier rejects a wrong anchor +//! and a wrong nullifier. +//! +//! Pre-existing bugs the green runs avoid, each with an `#[ignore]` reproducer below: +//! - F5a: `invalidate_block` panics while a send is pending (`nd.spent.unwrap()` on a note that +//! only has `unconfirmed_spent`). So no reorg happens while a send is pending. +//! - F5b: a send that never mines leaves its notes with a stale witness (pending notes are not +//! advanced); a reload drops `unconfirmed_spent` and exposes it as spendable. So no reload or +//! throwaway copy happens while a send is pending, and a run that leaves a send unmined only +//! mines blocks afterwards. +//! - F5b via a reorg (same root cause): once a send is mined its notes are spent, but a reorg +//! that undoes the mining block back to a height >= sent tip + 2 revives them with the stale +//! entries of the pending blocks. So reorg depths are adjusted to stay out of that window. +//! +//! Sizes: `oracle_default_*` (4 x 5 seeds x 300 blocks, three proving sends per seed); +//! `#[ignore]` `oracle_full_*` (8 x 125 seeds x 400 blocks, one proving send per seed) and +//! `oracle_soak_*` (2 x 10 seeds x 5000 blocks). Every run prints its seed. `T6_SEED=` +//! (decimal or 0x-hex) replaces the seed list of any of them with that one seed, and +//! `T6_BLOCKS=` overrides the block count; `oracle_env_seed` runs exactly that seed. +//! A replay is exact up to the first mined real send (proof randomness comes from OsRng). + +use std::cell::{Cell, RefCell}; +use std::cmp; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::panic::{self, AssertUnwindSafe}; +use std::time::Instant; + +use rand::rngs::StdRng; +use rand::{Rng, RngCore, SeedableRng}; +use threadpool::ThreadPool; + +use bellman::groth16::{prepare_verifying_key, PreparedVerifyingKey, Proof, VerifyingKey}; +use bip39::{Language, Mnemonic}; +use ff::{Field, PrimeField, PrimeFieldRepr}; +use pairing::bls12_381::{Bls12, Fr}; +use protobuf::Message; + +use zcash_client_backend::encoding::{ + encode_extended_full_viewing_key, encode_extended_spending_key, encode_payment_address, +}; +use zcash_client_backend::proto::compact_formats::{CompactBlock, CompactOutput, CompactSpend, CompactTx}; +use zcash_primitives::block::BlockHash; +use zcash_primitives::jubjub::fs::Fs; +use zcash_primitives::keys::OutgoingViewingKey; +use zcash_primitives::merkle_tree::{CommitmentTree, CommitmentTreeWitness, Hashable, IncrementalWitness}; +use zcash_primitives::note_encryption::{try_sapling_note_decryption, Memo, SaplingNoteEncryption}; +use zcash_primitives::primitives::{Note, PaymentAddress}; +use zcash_primitives::redjubjub::PublicKey; +use zcash_primitives::sapling::Node; +use zcash_primitives::transaction::components::amount::DEFAULT_FEE; +use zcash_primitives::transaction::{signature_hash, Transaction, TxId, SIGHASH_ALL}; +use zcash_primitives::zip32::{DiversifierIndex, ExtendedFullViewingKey, ExtendedSpendingKey}; +use zcash_primitives::JUBJUB; +use zcash_proofs::sapling::SaplingVerificationContext; + +use super::{LightWallet, MAX_REORG}; +use crate::lightclient::LightClientConfig; +use crate::SaplingParams; + +/// Sapling consensus branch id used for building and verifying sends. +const BRANCH_ID: u32 = 0x76b8_09bb; +/// Builder::new sets expiry = height + 20, and cleanup_mempool drops a send 20 blocks after it. +const TX_EXPIRY_DELTA: i32 = 20; +/// The reference keeps blocks and witness history this far below the tip (> MAX_REORG + 1). +const KEEP: i32 = MAX_REORG as i32 + 10; +/// What the broadcast closure returns. Nothing is ever sent. +const NOT_BROADCAST: &str = "oracle: captured, not broadcast"; + +fn cfg(anchor_offset: u32) -> LightClientConfig { + LightClientConfig { + server: "0.0.0.0:0".parse().unwrap(), + chain_name: "main".to_string(), // "test" panics in base58_pubkey_address + sapling_activation_height: 0, + consensus_branch_id: "76b809bb".to_string(), + anchor_offset, + no_cert_verification: false, + data_dir: None, // a LightWallet never touches the disk itself + } +} + +fn wbytes(w: &IncrementalWitness) -> Vec { + let mut v = vec![]; + w.write(&mut v).unwrap(); + v +} + +fn tbytes(t: &CommitmentTree) -> Vec { + let mut v = vec![]; + t.write(&mut v).unwrap(); + v +} + +fn hash_hex(h: &BlockHash) -> String { + let mut b = h.0.to_vec(); + b.reverse(); + hex::encode(b) +} + +fn nf32(v: &[u8]) -> [u8; 32] { + let mut a = [0u8; 32]; + a.copy_from_slice(v); + a +} + +/// Hash `leaf` up an authentication path (bool = the sibling is the left child). +fn path_root(path: &CommitmentTreeWitness, leaf: Node) -> Node { + let mut cur = leaf; + for (depth, p) in path.auth_path.iter().enumerate() { + let (sib, sib_is_left) = p.expect("complete authentication path"); + cur = if sib_is_left { + Node::combine(depth, &sib, &cur) + } else { + Node::combine(depth, &cur, &sib) + }; + } + cur +} + +// --------------------------------------------------------------------------------------------- +// Offline verifier +// --------------------------------------------------------------------------------------------- + +struct Verifier { + spend: PreparedVerifyingKey, + output: PreparedVerifyingKey, +} + +impl Verifier { + fn new() -> Self { + // A Groth16 parameters file starts with its verifying key. + let read = |name: &str| { + let p = SaplingParams::get(name).expect("embedded Sapling params"); + let vk = VerifyingKey::::read(&p[..]).expect("verifying key of the params file"); + prepare_verifying_key(&vk) + }; + Verifier { + spend: read("sapling-spend.params"), + output: read("sapling-output.params"), + } + } +} + +fn sighash_of(tx: &Transaction) -> [u8; 32] { + nf32(&signature_hash(tx, BRANCH_ID, SIGHASH_ALL, None)) +} + +fn check_spend_with(v: &Verifier, tx: &Transaction, i: usize, anchor: Fr, nf: [u8; 32]) -> bool { + let s = &tx.shielded_spends[i]; + let proof = match Proof::::read(&s.zkproof[..]) { + Ok(p) => p, + Err(_) => return false, + }; + let sig = match s.spend_auth_sig { + Some(sig) => sig, + None => return false, + }; + let mut c = SaplingVerificationContext::new(); + c.check_spend(s.cv.clone(), anchor, &nf, PublicKey(s.rk.0.clone()), &sighash_of(tx), sig, + proof, &v.spend, &JUBJUB) +} + +/// Parse and fully verify the Sapling part of a wallet-built tx. +fn verify_tx(v: &Verifier, raw: &[u8], anchor: Fr, eligible: &HashSet<[u8; 32]>, + value_balance: i64, expiry: u32, ctx: &str) -> Transaction { + let tx = Transaction::read(raw).unwrap_or_else(|e| panic!("{}: sent tx does not parse: {}", ctx, e)); + let mut again = vec![]; + tx.write(&mut again).unwrap(); + assert!(again == raw, "{}: sent tx does not re-serialize to the same bytes", ctx); + assert!(tx.vin.is_empty() && tx.vout.is_empty() && tx.joinsplits.is_empty(), + "{}: unexpected transparent or sprout parts", ctx); + assert_eq!(tx.expiry_height, expiry, "{}: expiry height", ctx); + assert_eq!(i64::from(tx.value_balance), value_balance, "{}: value balance", ctx); + assert!(!tx.shielded_spends.is_empty(), "{}: a shielded send without spends", ctx); + + let sighash = sighash_of(&tx); + let mut seen = HashSet::new(); + let mut vctx = SaplingVerificationContext::new(); + for (i, s) in tx.shielded_spends.iter().enumerate() { + assert!(s.anchor == anchor, + "{}: spend {} anchor {:?} is not the reference root at tip-anchor_offset {:?}", + ctx, i, s.anchor, anchor); + assert!(eligible.contains(&s.nullifier), + "{}: spend {} nullifier {} is not an eligible unspent note", ctx, i, hex::encode(s.nullifier)); + assert!(seen.insert(s.nullifier), "{}: spend {} repeats a nullifier", ctx, i); + let proof = Proof::::read(&s.zkproof[..]).expect("spend proof parses"); + let sig = s.spend_auth_sig.expect("spend auth sig"); + assert!(vctx.check_spend(s.cv.clone(), s.anchor, &s.nullifier, PublicKey(s.rk.0.clone()), + &sighash, sig, proof, &v.spend, &JUBJUB), + "{}: spend {} fails check_spend", ctx, i); + } + for (i, o) in tx.shielded_outputs.iter().enumerate() { + let proof = Proof::::read(&o.zkproof[..]).expect("output proof parses"); + assert!(vctx.check_output(o.cv.clone(), o.cmu, o.ephemeral_key.clone(), proof, &v.output, &JUBJUB), + "{}: output {} fails check_output", ctx, i); + } + assert!(vctx.final_check(tx.value_balance, &sighash, tx.binding_sig.expect("binding sig"), &JUBJUB), + "{}: binding signature / value balance fails final_check", ctx); + tx +} + +// --------------------------------------------------------------------------------------------- +// Generator and reference +// --------------------------------------------------------------------------------------------- + +#[derive(Clone, Copy, PartialEq, Debug)] +enum KeyKind { + Hd, + Sk, + Vk, +} + +#[derive(Clone)] +struct Key { + fvk: ExtendedFullViewingKey, + has_sk: bool, + kind: KeyKind, +} + +fn keys_of(w: &LightWallet) -> Vec { + w.zkeys.read().unwrap().iter().map(|zk| { + assert_eq!(zk.extsk.is_some(), zk.have_spending_key(), "unencrypted wallet key state"); + Key { + fvk: zk.extfvk.clone(), + has_sk: zk.extsk.is_some(), + kind: if zk.hdkey_num.is_some() { + KeyKind::Hd + } else if zk.extsk.is_some() { + KeyKind::Sk + } else { + KeyKind::Vk + }, + } + }).collect() +} + +#[derive(Clone)] +struct Owned { + account: usize, + note: Note, + to: PaymentAddress, +} + +/// A tx as mined: its compact form plus which outputs are ours. Kept so it can be re-mined +/// on the branch that replaces it after a reorg. +#[derive(Clone)] +struct GenTx { + ctx: CompactTx, + ours: Vec>, + /// Raw bytes of a wallet-built tx (for scan_full_tx). + raw: Option>, + /// A wallet-built tx is only valid on a branch that keeps its anchor block. + anchor_height: Option, +} + +impl GenTx { + fn txid(&self) -> TxId { + TxId(nf32(&self.ctx.hash)) + } + + fn nfs(&self) -> Vec<[u8; 32]> { + self.ctx.spends.iter().map(|s| nf32(&s.nf)).collect() + } +} + +struct RefBlock { + height: i32, + hash: BlockHash, + tree: CommitmentTree, + root: Node, +} + +struct RefNote { + txid: TxId, + account: usize, + to: PaymentAddress, + addr: String, + value: u64, + has_sk: bool, + position: u64, + cmu: Node, + nf: [u8; 32], + recv: i32, + is_change: bool, + /// True witness at the tip while the note is unspent (frozen once it is spent). + witness: IncrementalWitness, + /// True witness at the end of each block while the note was unspent. + hist: VecDeque<(i32, IncrementalWitness)>, + /// Legacy witness-list length model. + len: usize, + /// (spending txid, block of the spending tx) + spent: Option<(TxId, i32)>, + pending: Option, + /// Tip at which a wallet send made this note pending (diagnostics). + was_pending: Option, +} + +impl RefNote { + fn live(&self) -> bool { + self.spent.is_none() && self.pending.is_none() + } + + fn hist_at(&self, h: i32) -> Option<&IncrementalWitness> { + self.hist.iter().rev().find(|(hh, _)| *hh == h).map(|(_, w)| w) + } +} + +struct Pending { + gtx: GenTx, + txid: TxId, + sent_tip: i32, +} + +#[derive(Clone, Copy, PartialEq)] +enum SendKind { + /// On a throwaway copy (`read(write(w))`) built with this anchor offset; never mined. + Probe(u32), + /// On the wallet itself with its own anchor offset; pending until mined (or never). + Real, +} + +#[derive(Default)] +struct Stats { + blocks: usize, + outputs: usize, + ours: usize, + spends: usize, + reorgs: usize, + max_depth: usize, + reloads: usize, + probes: usize, + sends: usize, + mined_sends: usize, + insufficient: usize, + requeued: usize, + remined: usize, + full_checks: usize, + t_gen: f64, + t_scan: f64, + t_ref: f64, + t_check: f64, + t_send: f64, + t_reorg: f64, + t_reload: f64, + t_full: f64, +} + +struct Sim { + seed: u64, + rng: StdRng, + cfg: LightClientConfig, + w: LightWallet, + pool: ThreadPool, + foreign: Vec<(OutgoingViewingKey, PaymentAddress)>, + + blocks: VecDeque, + /// Heights of the wallet's retained block window (legacy rule: at most MAX_REORG + 1). + win: VecDeque, + tree: CommitmentTree, + notes: Vec, + /// Wallet txs: txid -> (block, total_shielded_value_spent) + rel: HashMap, + /// Relevant txs per height on the current branch, for re-mining after a reorg. + mined: VecDeque<(i32, Vec)>, + requeue: Vec<(i32, GenTx)>, + /// Our sends in the wallet's mempool: txid -> tip when sent. + mempool: HashMap, + pending: Option, + /// A send was left unmined: from here on only blocks are mined (F5a/F5b). + frozen: bool, + /// (tip when sent, block that first mined it) of our mined sends: a reorg to a height in + /// (sent + 1, mined] would revive their notes with stale witnesses (F5b via a reorg). + pending_windows: Vec<(i32, i32)>, + /// Blocks still to be paid to us (setup helper). + force_pay: usize, + force_output: bool, + controls_done: bool, + target_live: usize, + + step: usize, + op: &'static str, + cleared_seen: Cell, + stats: Stats, +} + +impl Sim { + fn new(seed: u64) -> Sim { + let mut rng = StdRng::seed_from_u64(seed); + let cfg = cfg(if rng.gen_bool(0.5) { 0 } else { 4 }); + + let mut entropy = [0u8; 32]; + rng.fill_bytes(&mut entropy); + let phrase = Mnemonic::from_entropy(&entropy, Language::English).unwrap().phrase().to_string(); + let n_hd = rng.gen_range(0, 4); + let mut w = LightWallet::new(Some(phrase), &cfg, 0, n_hd).unwrap(); + + if rng.gen_bool(0.6) { + let mut s = [0u8; 32]; + rng.fill_bytes(&mut s); + let sk = encode_extended_spending_key(cfg.hrp_sapling_private_key(), &ExtendedSpendingKey::master(&s)); + let r = w.add_imported_sk(sk, 0); + assert!(!r.starts_with("Error"), "import sk: {}", r); + } + if rng.gen_bool(0.6) { + let mut s = [0u8; 32]; + rng.fill_bytes(&mut s); + let fvk = ExtendedFullViewingKey::from(&ExtendedSpendingKey::master(&s)); + let vk = encode_extended_full_viewing_key(cfg.hrp_sapling_viewing_key(), &fvk); + let r = w.add_imported_vk(vk, 0); + assert!(!r.starts_with("Error"), "import vk: {}", r); + } + + let foreign = (0..3).map(|_| { + let mut s = [0u8; 32]; + rng.fill_bytes(&mut s); + let fvk = ExtendedFullViewingKey::from(&ExtendedSpendingKey::master(&s)); + (fvk.fvk.ovk, fvk.default_address().unwrap().1) + }).collect(); + + // Start from a real main-chain checkpoint, so leaf positions are realistic. + let want = rng.gen_range(50u64, 311) * 10_000; + let (h, hash_hex_s, tree_hex) = cfg.get_initial_state(want).expect("main checkpoint"); + let (h, hash_hex_s, tree_hex) = (h as i32, hash_hex_s.to_string(), tree_hex.to_string()); + assert!(w.set_initial_block(h, &hash_hex_s, &tree_hex)); + let tree = CommitmentTree::::read(&hex::decode(&tree_hex).unwrap()[..]).unwrap(); + let mut hb = hex::decode(&hash_hex_s).unwrap(); + hb.reverse(); + let mut blocks = VecDeque::new(); + let root = tree.root(); + blocks.push_back(RefBlock { height: h, hash: BlockHash::from_slice(&hb), tree: tree.clone(), root }); + let mut win = VecDeque::new(); + win.push_back(h); + + let sim = Sim { + seed, + rng, + cfg, + w, + pool: ThreadPool::new(2), + foreign, + blocks, + win, + tree, + notes: vec![], + rel: HashMap::new(), + mined: VecDeque::new(), + requeue: vec![], + mempool: HashMap::new(), + pending: None, + frozen: false, + pending_windows: vec![], + force_pay: 0, + force_output: false, + controls_done: false, + target_live: 8, + step: 0, + op: "init", + cleared_seen: Cell::new(0), + stats: Stats::default(), + }; + sim.check(true); + sim + } + + fn ctx(&self) -> String { + format!("[oracle seed={:#x} step={} op={} tip={}]", self.seed, self.step, self.op, self.tip()) + } + + fn tip(&self) -> i32 { + self.blocks.back().unwrap().height + } + + fn block_at(&self, h: i32) -> &RefBlock { + self.blocks.iter().rev().find(|b| b.height == h) + .unwrap_or_else(|| panic!("{} reference block {} not kept", self.ctx(), h)) + } + + /// (tip, effective anchor offset), as LightWallet::get_target_height_and_anchor_offset. + fn anchor(&self, offset_cfg: u32) -> (i32, i32) { + let tip = self.tip(); + let anchor_h = cmp::max(tip - offset_cfg as i32, *self.win.front().unwrap()); + (tip, tip - anchor_h) + } + + fn rint(&mut self, lo: u64, hi_incl: u64) -> u64 { + self.rng.gen_range(lo, hi_incl + 1) + } + + fn gen_value(&mut self) -> u64 { + match self.rng.gen_range(0, 100) { + 0..=7 => 0, // HushChat-style zero-value note + 8..=9 => self.rint(1, 9_999), // dust below the fee + 10..=97 => self.rint(20_000, 400_000), + _ => self.rint(100_000_000, 1_000_000_000), + } + } + + fn make_output(&mut self, ovk: OutgoingViewingKey, to: &PaymentAddress, value: u64, bad_epk: bool) + -> (CompactOutput, Note) { + let note = Note { + g_d: to.diversifier.g_d::(&JUBJUB).unwrap(), + pk_d: to.pk_d.clone(), + value, + r: Fs::random(&mut self.rng), + }; + let enc = SaplingNoteEncryption::new(ovk, note.clone(), to.clone(), Memo::default(), &mut self.rng); + let mut cmu = vec![]; + note.cm(&JUBJUB).into_repr().write_le(&mut cmu).unwrap(); + let mut epk = vec![]; + if bad_epk { + // Unparseable epk: the wallet must still append the cmu to the tree. + epk.resize(31, 0); + self.rng.fill_bytes(&mut epk); + } else { + enc.epk().write(&mut epk).unwrap(); + } + let ct = enc.encrypt_note_plaintext(); + let mut co = CompactOutput::new(); + co.set_cmu(cmu); + co.set_epk(epk); + co.set_ciphertext(ct[..52].to_vec()); + (co, note) + } + + fn our_output(&mut self, keys: &[Key], account: usize, value: u64, diversified: bool) + -> (CompactOutput, Owned) { + let fvk = keys[account].fvk.clone(); + let to = if diversified { + let j = self.rng.gen_range(1u8, 250); + let mut idx = DiversifierIndex::new(); + idx.0[0] = j; + fvk.address(idx).unwrap().1 + } else { + fvk.default_address().unwrap().1 + }; + let (co, note) = self.make_output(fvk.fvk.ovk, &to, value, false); + (co, Owned { account, note, to }) + } + + fn pick_account(&mut self, keys: &[Key]) -> usize { + let hd: Vec = (0..keys.len()).filter(|&i| keys[i].kind == KeyKind::Hd).collect(); + let sk: Vec = (0..keys.len()).filter(|&i| keys[i].kind == KeyKind::Sk).collect(); + let vk: Vec = (0..keys.len()).filter(|&i| keys[i].kind == KeyKind::Vk).collect(); + let r = self.rng.gen_range(0, 100); + if r < 35 { + 0 + } else if r < 70 { + // Mostly the first HD keys, so the gap rule only grows the key list a little. + let span = if self.rng.gen_bool(0.95) { cmp::min(hd.len(), 3) } else { hd.len() }; + hd[self.rng.gen_range(0, span)] + } else if r < 85 && !sk.is_empty() { + sk[self.rng.gen_range(0, sk.len())] + } else if !vk.is_empty() { + vk[self.rng.gen_range(0, vk.len())] + } else { + 0 + } + } + + fn gen_tx(&mut self, h: i32, keys: &[Key], n_out: usize, used: &mut HashSet<[u8; 32]>) -> GenTx { + let mut ctx = CompactTx::new(); + let mut txid = vec![0u8; 32]; + self.rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + let mut ours = vec![]; + + // Compact spends of our notes (including view-only ones, spent by their real owner). + let cands: Vec = (0..self.notes.len()) + .filter(|&i| { + let n = &self.notes[i]; + n.live() && n.recv < h && !used.contains(&n.nf) + }) + .collect(); + // Spend pressure keeps the live-note count near a per-seed target (8..=40), so both + // small (< 16) and large witness sets occur. + let excess = cands.len() as f64 - self.target_live as f64; + let p_spend = (0.1 + excess * 0.06).max(0.04).min(0.95); + let mut change_account = None; + if !cands.is_empty() && self.rng.gen_bool(p_spend) { + let want = if excess > 20.0 { 3 } else if excess > 8.0 || self.rng.gen_bool(0.2) { 2 } else { 1 }; + let k = cmp::min(want, cands.len()); + let mut picked: Vec = vec![]; + while picked.len() < k { + let i = cands[self.rng.gen_range(0, cands.len())]; + if !picked.contains(&i) { + picked.push(i); + } + } + for i in picked { + let nf = self.notes[i].nf; + let mut cs = CompactSpend::new(); + cs.set_nf(nf.to_vec()); + ctx.spends.push(cs); + used.insert(nf); + if change_account.is_none() { + change_account = Some(self.notes[i].account); + } + } + self.stats.spends += k; + } + if self.rng.gen_bool(0.1) { + let mut nf = vec![0u8; 32]; + self.rng.fill_bytes(&mut nf); + let mut cs = CompactSpend::new(); + cs.set_nf(nf); + ctx.spends.push(cs); + } + + let n_out = if change_account.is_some() { cmp::max(n_out, 1) } else { n_out }; + for j in 0..n_out { + if j == 0 && change_account.is_some() && self.rng.gen_bool(0.75) { + let v = self.gen_value(); + let (co, o) = self.our_output(keys, change_account.unwrap(), v, false); + ctx.outputs.push(co); + ours.push(Some(o)); + } else if self.rng.gen_range(0, 100) < 18 { + let a = self.pick_account(keys); + let v = self.gen_value(); + let div = self.rng.gen_bool(0.05); + let (co, o) = self.our_output(keys, a, v, div); + ctx.outputs.push(co); + ours.push(Some(o)); + } else { + let (ovk, to) = self.foreign[self.rng.gen_range(0, self.foreign.len())].clone(); + let v = self.gen_value(); + let bad = self.rng.gen_bool(0.04); + let (co, _) = self.make_output(ovk, &to, v, bad); + ctx.outputs.push(co); + ours.push(None); + } + } + GenTx { ctx, ours, raw: None, anchor_height: None } + } + + fn gen_txs(&mut self, h: i32, keys: &[Key], already: &[GenTx]) -> Vec { + let mut used: HashSet<[u8; 32]> = already.iter().flat_map(|g| g.nfs()).collect(); + let mut out = vec![]; + if self.force_pay > 0 { + self.force_pay -= 1; + let mut ctx = CompactTx::new(); + let mut txid = vec![0u8; 32]; + self.rng.fill_bytes(&mut txid); + ctx.set_hash(txid); + let v = self.rint(100_000, 400_000); + let (co, o) = self.our_output(keys, 0, v, false); + ctx.outputs.push(co); + out.push(GenTx { ctx, ours: vec![Some(o)], raw: None, anchor_height: None }); + } + let shape = self.rng.gen_range(0, 100); + let (ntx, max_out): (usize, usize) = match shape { + 0..=44 => (0, 0), + 45..=89 => (self.rint(1, 3) as usize, 4), + 90..=97 => (self.rint(4, 8) as usize, 6), + _ => (1, 50), + }; + let ntx = if self.force_output { cmp::max(ntx, 1) } else { ntx }; + for _ in 0..ntx { + let n_out = if max_out == 50 { + 50 + } else if self.force_output { + self.rint(1, cmp::max(max_out, 1) as u64) as usize + } else { + self.rint(0, max_out as u64) as usize + }; + let g = self.gen_tx(h, keys, n_out, &mut used); + out.push(g); + } + out + } + + /// A requeued tx may only be re-mined if it does not double-spend. + fn still_valid(&self, g: &GenTx, in_block: &[GenTx]) -> bool { + let used: HashSet<[u8; 32]> = in_block.iter().flat_map(|x| x.nfs()).collect(); + g.nfs().iter().all(|nf| { + !used.contains(nf) && self.notes.iter().find(|n| n.nf == *nf).map_or(true, |n| n.live()) + }) + } + + /// Mine one block: requeued txs that are due, `extra`, and (optionally) random txs. + fn mine(&mut self, extra: Vec, random_txs: bool) { + self.op = "mine"; + let h = self.tip() + 1; + let keys = keys_of(&self.w); + + let mut txs: Vec = vec![]; + let mut later = vec![]; + for (at, g) in std::mem::replace(&mut self.requeue, vec![]) { + if at <= h { + if self.still_valid(&g, &txs) { + txs.push(g); + self.stats.remined += 1; + } + } else { + later.push((at, g)); + } + } + self.requeue = later; + txs.extend(extra); + let t = Instant::now(); + if random_txs { + let more = self.gen_txs(h, &keys, &txs); + txs.extend(more); + } + for (i, g) in txs.iter_mut().enumerate() { + g.ctx.set_index(i as u64); + } + + let mut cb = CompactBlock::new(); + cb.set_height(h as u64); + let mut hash = vec![0u8; 32]; + self.rng.fill_bytes(&mut hash); + cb.set_hash(hash.clone()); + cb.set_prevHash(self.blocks.back().unwrap().hash.0.to_vec()); + cb.set_time(1_600_000_000 + h as u32); + for g in &txs { + cb.vtx.push(g.ctx.clone()); + } + let bytes = cb.write_to_bytes().unwrap(); + + self.stats.t_gen += t.elapsed().as_secs_f64(); + let t = Instant::now(); + let got = self.w.scan_block_with_pool(&bytes, &self.pool); + self.stats.t_scan += t.elapsed().as_secs_f64(); + let t = Instant::now(); + let want = self.ref_apply(h, BlockHash::from_slice(&hash), &txs, &keys); + self.stats.t_ref += t.elapsed().as_secs_f64(); + assert!(got == Ok(want.clone()), "{} scan_block returned {:?}, expected Ok({:?})", self.ctx(), got, want); + + // Full-tx scan of mined wallet-built txs (memos, outgoing metadata). + for g in &txs { + if let Some(raw) = &g.raw { + let tx = Transaction::read(&raw[..]).unwrap(); + self.w.scan_full_tx(&tx, h, 0); + let txs_w = self.w.txs.read().unwrap(); + let wtx = txs_w.get(&g.txid()).unwrap_or_else(|| panic!("{} mined send not in txs", self.ctx())); + assert!(wtx.full_tx_scanned, "{} mined send not marked full_tx_scanned", self.ctx()); + self.stats.mined_sends += 1; + } + } + self.stats.blocks += 1; + let t = Instant::now(); + self.check(false); + self.stats.t_check += t.elapsed().as_secs_f64(); + } + + /// Apply a block to the reference. Returns what scan_block must return. + fn ref_apply(&mut self, h: i32, hash: BlockHash, txs: &[GenTx], keys: &[Key]) -> Vec { + // The wallet tracks the nullifiers of every note not confirmed spent at block start + // (pending ones included). The reference advances the true witness of that same set + // (the wallet skips pending notes, F5b). + let tracked: HashMap<[u8; 32], usize> = (0..self.notes.len()) + .filter(|&i| self.notes[i].spent.is_none()) + .map(|i| (self.notes[i].nf, i)) + .collect(); + let advancing: Vec = tracked.values().cloned().collect(); + let first_new = self.notes.len(); + let mut any_rel = false; + + for g in txs { + let txid = g.txid(); + let mut rel = false; + let mut spent_value = 0u64; + let mut spent_accounts = HashSet::new(); + for nf in g.nfs() { + if let Some(&i) = tracked.get(&nf) { + assert!(self.notes[i].spent.is_none(), "{} the generator double-spent a note", self.ctx()); + let n = &mut self.notes[i]; + n.spent = Some((txid, h)); + n.pending = None; + spent_accounts.insert(n.account); + spent_value += n.value; + rel = true; + } + } + for (j, out) in g.ctx.outputs.iter().enumerate() { + let cmu = out.cmu().expect("the generator only emits parseable cmus"); + let node = Node::new(cmu.into_repr()); + self.tree.append(node).unwrap(); + for &i in &advancing { + self.notes[i].witness.append(node).unwrap(); + } + for i in first_new..self.notes.len() { + self.notes[i].witness.append(node).unwrap(); + } + self.stats.outputs += 1; + if let Some(o) = &g.ours[j] { + let position = (self.tree.size() - 1) as u64; + let fvk = &keys[o.account].fvk; + let nf = nf32(&o.note.nf(&fvk.fvk.vk, position, &JUBJUB)); + self.notes.push(RefNote { + txid, + account: o.account, + to: o.to.clone(), + addr: encode_payment_address(self.cfg.hrp_sapling_address(), &o.to), + value: o.note.value, + has_sk: keys[o.account].has_sk, + position, + cmu: node, + nf, + recv: h, + is_change: spent_accounts.contains(&o.account), + witness: IncrementalWitness::from_tree(&self.tree), + hist: VecDeque::new(), + len: 0, + spent: None, + pending: None, + was_pending: None, + }); + self.stats.ours += 1; + rel = true; + } + } + if rel { + any_rel = true; + self.rel.insert(txid, (h, spent_value)); + } + } + + // End of block: legacy length rule, history, window. + for i in 0..first_new { + let n = &mut self.notes[i]; + if n.len > 0 { + n.len = cmp::min(n.len + 1, MAX_REORG); + } + } + for i in first_new..self.notes.len() { + self.notes[i].len = 1; + } + for &i in advancing.iter().chain((first_new..self.notes.len()).collect::>().iter()) { + let n = &mut self.notes[i]; + n.hist.push_back((h, n.witness.clone())); + } + let root = self.tree.root(); + self.blocks.push_back(RefBlock { height: h, hash, tree: self.tree.clone(), root }); + while self.blocks.front().unwrap().height < h - KEEP { + self.blocks.pop_front(); + } + self.win.push_back(h); + while self.win.len() > MAX_REORG + 1 { + self.win.pop_front(); + } + let win_first = *self.win.front().unwrap(); + self.pending_windows.retain(|(_, m)| *m >= win_first); + for n in self.notes.iter_mut() { + while n.hist.front().map_or(false, |(hh, _)| *hh < h - KEEP) { + n.hist.pop_front(); + } + // A note whose spender is below the window can never be un-spent again. + if n.spent.map_or(false, |(_, s)| s < win_first) && !n.hist.is_empty() { + n.hist = VecDeque::new(); + } + } + + let ids: HashSet = txs.iter().map(|g| g.txid()).collect(); + self.mempool.retain(|t, b| !ids.contains(t) && h < *b + TX_EXPIRY_DELTA); + + let rel_txs: Vec = txs.iter().filter(|g| self.rel.get(&g.txid()).map_or(false, |r| r.0 == h)) + .cloned().collect(); + self.mined.push_back((h, rel_txs)); + while self.mined.front().map_or(false, |(hh, _)| *hh < h - KEEP) { + self.mined.pop_front(); + } + + if any_rel { + txs.iter().map(|g| g.txid()).collect() + } else { + vec![] + } + } + + /// Reorg: pop the newest k blocks (one invalidate_block call, or one call per block like + /// the sync loop does), then let the new branch be generated. + fn reorg(&mut self, k: usize, stepwise: bool) { + assert!(self.pending.is_none() && !self.frozen, "{} reorg with a pending send (F5a)", self.ctx()); + assert!(k >= 1 && k < self.win.len(), "{} bad reorg depth {}", self.ctx(), k); + self.op = "reorg"; + let at = self.tip() - k as i32 + 1; + let n = if stepwise { + (0..k).map(|_| self.w.invalidate_block(self.w.last_scanned_height())).sum::() + } else { + self.w.invalidate_block(at) + }; + assert_eq!(n, k as u64, "{} invalidate_block({}) return value", self.ctx(), at); + self.ref_invalidate(at, k); + self.stats.reorgs += 1; + self.stats.max_depth = cmp::max(self.stats.max_depth, k); + self.check(true); + } + + fn ref_invalidate(&mut self, at: i32, k: usize) { + while self.blocks.back().unwrap().height >= at { + self.blocks.pop_back(); + } + self.tree = self.blocks.back().unwrap().tree.clone(); + let mut popped = 0; + while self.win.back().map_or(false, |h| *h >= at) { + self.win.pop_back(); + popped += 1; + } + assert_eq!(popped, k, "{} window pop count", self.ctx()); + assert!(!self.win.is_empty()); + + self.notes.retain(|n| n.recv < at); + let ctx = self.ctx(); + for n in self.notes.iter_mut() { + if n.spent.map_or(false, |(_, s)| s >= at) { + n.spent = None; + } + n.len = n.len.saturating_sub(k); + while n.hist.back().map_or(false, |(hh, _)| *hh >= at) { + n.hist.pop_back(); + } + if n.spent.is_none() { + let (hh, w) = n.hist.back().unwrap_or_else(|| panic!("{} no history for a live note", ctx)); + assert_eq!(*hh, at - 1, "{} history gap", ctx); + n.witness = w.clone(); + } + } + self.rel.retain(|_, r| r.0 < at); + self.pending_windows.retain(|(_, m)| *m < at); + + // The new branch may re-mine the dropped spends at other heights, or drop them. + let mut dropped = vec![]; + while self.mined.back().map_or(false, |(hh, _)| *hh >= at) { + let (_, gs) = self.mined.pop_back().unwrap(); + dropped.extend(gs); + } + dropped.reverse(); + // A requeued wallet-built tx whose anchor block was just popped is invalid now. + self.requeue.retain(|(_, g)| g.anchor_height.map_or(true, |a| a < at)); + for g in dropped { + let anchor_ok = g.anchor_height.map_or(true, |a| a < at); + if anchor_ok && self.rng.gen_bool(0.6) { + let when = at + self.rng.gen_range(0, k as i32 + 3); + self.requeue.push((when, g)); + self.stats.requeued += 1; + } + } + } + + fn pick_depth(&mut self) -> usize { + let max = self.win.len() - 1; // keep at least one block + let d: u64 = match self.rng.gen_range(0, 100) { + 0..=59 => self.rint(1, 3), + 60..=84 => self.rint(4, 20), + _ => self.rint(21, 100), + }; + cmp::min(d as usize, max) + } + + /// Adjust a reorg depth so the reorg does not undo the mining of one of our sends back into + /// the blocks where it was pending (F5b via a reorg, see its #[ignore] reproducer): prefer + /// keeping the mining block, else pop the whole pending window. + fn safe_depth(&self, k: usize) -> Option { + let tip = self.tip(); + let max = self.win.len() as i32 - 1; + let mut at = tip - k as i32 + 1; + for _ in 0..8 { + match self.pending_windows.iter().find(|(t, m)| *t + 1 < at && at <= *m) { + None => { + let k = tip - at + 1; + return if k >= 1 && k <= max { Some(k as usize) } else { None }; + } + Some(&(t, m)) => at = if m + 1 <= tip { m + 1 } else { t + 1 }, + } + } + None + } + + fn reload(&mut self) { + assert!(self.pending.is_none() && !self.frozen, "{} reload with a pending send (F5b)", self.ctx()); + self.op = "reload"; + self.reload_raw(); + self.stats.reloads += 1; + self.check(true); + if self.stats.reloads == 1 { + self.check_addresses(&self.w); // slow: the wallet derives every note's address per call + } + } + + /// write -> read -> write -> read; the second round trip must be byte-identical. + fn reload_raw(&mut self) { + let mut b1 = vec![]; + self.w.write(&mut b1).unwrap(); + let w2 = LightWallet::read(&b1[..], &self.cfg).unwrap(); + let mut b2 = vec![]; + w2.write(&mut b2).unwrap(); + let w3 = LightWallet::read(&b2[..], &self.cfg).unwrap(); + let mut b3 = vec![]; + w3.write(&mut b3).unwrap(); + assert!(b2 == b3, "{} write(read(x)) is not idempotent", self.ctx()); + self.w = w3; + } + + /// Build a send. Probe: on a throwaway copy with the given anchor offset. Real: on the + /// wallet itself, which leaves it pending. Returns whether a proving send was made (false + /// when there were not enough eligible funds; the error path is checked instead). + fn send(&mut self, v: &Verifier, kind: SendKind) -> bool { + assert!(self.pending.is_none() && !self.frozen, "{} send with a pending send", self.ctx()); + self.op = match kind { SendKind::Probe(_) => "probe-send", SendKind::Real => "send" }; + let offset_cfg = match kind { SendKind::Probe(o) => o, SendKind::Real => self.cfg.anchor_offset }; + let (tip, off) = self.anchor(offset_cfg); + let elig: Vec = (0..self.notes.len()) + .filter(|&i| { + let n = &self.notes[i]; + n.live() && n.has_sk && n.len >= off as usize + 1 + }) + .collect(); + let total: u64 = elig.iter().map(|&i| self.notes[i].value).sum(); + let fee = u64::from(DEFAULT_FEE); + // Notes that cover the fee on their own. The amount stays below the smallest of them, so + // any selection order stops after one such note (plus any dust/zero notes before it). + let mut covering: Vec = elig.iter().map(|&i| self.notes[i].value).filter(|&x| x > fee).collect(); + covering.sort(); + + let keys = keys_of(&self.w); + let self_send = self.rng.gen_bool(0.2); + let to = if self_send { + let a = self.pick_account(&keys); + encode_payment_address(self.cfg.hrp_sapling_address(), &keys[a].fvk.default_address().unwrap().1) + } else { + let (_, pa) = &self.foreign[self.rng.gen_range(0, self.foreign.len())]; + encode_payment_address(self.cfg.hrp_sapling_address(), pa) + }; + let memo = if self.rng.gen_bool(0.5) { + Some(format!("oracle {:#x} {}", self.seed, self.step)) + } else { + None + }; + + if covering.is_empty() { + // Not enough: check the error path instead. + self.insufficient_check(); + return false; + } + let (ss, so) = (SaplingParams::get("sapling-spend.params").unwrap(), + SaplingParams::get("sapling-output.params").unwrap()); + + // Proving cost is per input, so keep the worst case over all selection orders bounded. + let values: Vec = elig.iter().map(|&i| self.notes[i].value).collect(); + let max1 = *covering.last().unwrap(); + let multi = if covering.len() >= 2 && self.rng.gen_bool(0.5) { + // More than the largest note: needs at least two inputs. + let amount = self.rint(max1 - fee + 1, cmp::min(total, max1 + 100_000) - fee); + if worst_case_inputs(&values, amount + fee) <= 10 { Some(amount) } else { None } + } else { + None + }; + let amount = match multi { + Some(a) => a, + None => self.rint(1, covering[0] - fee), + }; + let tos: Vec<(&str, u64, Option)> = if amount >= 2 && self.rng.gen_bool(0.15) { + let a1 = self.rint(1, amount - 1); + vec![(to.as_str(), a1, memo.clone()), (to.as_str(), amount - a1, None)] + } else { + vec![(to.as_str(), amount, memo)] + }; + + let copy; + let wallet: &LightWallet = match kind { + SendKind::Probe(o) => { + let mut b = vec![]; + self.w.write(&mut b).unwrap(); + copy = LightWallet::read(&b[..], &cfg(o)).unwrap(); + self.check_wallet(©, o, true); + © + } + SendKind::Real => &self.w, + }; + let captured: RefCell>> = RefCell::new(None); + let t0 = Instant::now(); + let res = wallet.send_to_address(BRANCH_ID, &ss, &so, false, tos.clone(), &fee, |raw| { + *captured.borrow_mut() = Some(raw.to_vec()); + Ok(NOT_BROADCAST.to_string()) + }); + let (txid_s, raw) = res.unwrap_or_else(|e| { + panic!("{} send of {} (eligible {}) failed: {}", self.ctx(), amount, total, e) + }); + assert_eq!(txid_s, NOT_BROADCAST); + assert!(captured.borrow().as_ref() == Some(&raw), "{} returned bytes differ from the broadcast bytes", + self.ctx()); + + let eligible: HashSet<[u8; 32]> = elig.iter().map(|&i| self.notes[i].nf).collect(); + let anchor = Fr::from(self.block_at(tip - off).root); + let ctx = format!("{} offset={}", self.ctx(), off); + let tx = verify_tx(v, &raw, anchor, &eligible, fee as i64, (tip + TX_EXPIRY_DELTA) as u32, &ctx); + let txid = tx.txid(); + let tx_nfs: HashSet<[u8; 32]> = tx.shielded_spends.iter().map(|s| s.nullifier).collect(); + let selected: u64 = self.notes.iter().filter(|n| tx_nfs.contains(&n.nf)).map(|n| n.value).sum(); + assert!(selected >= amount + fee, "{} selected {} < {} + fee", ctx, selected, amount); + + if !self.controls_done { + // The verifier is not vacuous: a wrong anchor or nullifier must be rejected. + assert!(check_spend_with(v, &tx, 0, anchor, tx.shielded_spends[0].nullifier), "{} control", ctx); + if let Some(other) = self.blocks.iter().map(|b| Fr::from(b.root)).find(|r| *r != anchor) { + assert!(!check_spend_with(v, &tx, 0, other, tx.shielded_spends[0].nullifier), + "{} the verifier accepted a wrong anchor", ctx); + } + let mut bad_nf = tx.shielded_spends[0].nullifier; + bad_nf[0] ^= 1; + assert!(!check_spend_with(v, &tx, 0, anchor, bad_nf), "{} the verifier accepted a wrong nullifier", + ctx); + self.controls_done = true; + } + + // The sending wallet marks exactly the spent notes pending and records the send. + { + let txs = wallet.txs.read().unwrap(); + let mut marked = HashSet::new(); + for nd in txs.values().flat_map(|t| t.notes.iter()) { + if let Some(t) = nd.unconfirmed_spent { + assert_eq!(t, txid, "{} a note is pending for another tx", ctx); + marked.insert(nd.nullifier); + } + } + assert!(marked == tx_nfs, "{} pending notes differ from the spent nullifiers", ctx); + let mp = wallet.mempool_txs.read().unwrap(); + let e = mp.get(&txid).unwrap_or_else(|| panic!("{} send missing from mempool_txs", ctx)); + assert_eq!(e.block, tip, "{} mempool entry height", ctx); + assert_eq!(e.outgoing_metadata.len(), tos.len(), "{} mempool outgoing metadata", ctx); + } + self.stats.t_send += t0.elapsed().as_secs_f64(); + println!("{} built and verified a send: {} spends, {} outputs, offset {}, {:.1}s", + ctx, tx.shielded_spends.len(), tx.shielded_outputs.len(), off, t0.elapsed().as_secs_f64()); + + match kind { + SendKind::Probe(_) => { + self.stats.probes += 1; + } + SendKind::Real => { + for n in self.notes.iter_mut() { + if tx_nfs.contains(&n.nf) { + n.pending = Some(txid); + n.was_pending = Some(tip); + } + } + self.mempool.insert(txid, tip); + let gtx = GenTx { + ctx: compact_of(&tx), + ours: owned_outputs(&tx, &keys), + raw: Some(raw), + anchor_height: Some(tip - off), + }; + self.pending = Some(Pending { gtx, txid, sent_tip: tip }); + self.stats.sends += 1; + self.check(false); + } + } + true + } + + fn check(&self, full: bool) { + self.check_wallet(&self.w, self.cfg.anchor_offset, full); + } + + /// Compare a wallet (the live one, or a copy built with `offset_cfg`) with the reference. + fn check_wallet(&self, w: &LightWallet, offset_cfg: u32, full: bool) { + let ctx = self.ctx(); + let tip = self.tip(); + + let (h, hash, tree) = w.get_sapling_tree().unwrap(); + assert_eq!(h, tip, "{} sapling tree height", ctx); + assert_eq!(hash, hash_hex(&self.blocks.back().unwrap().hash), "{} tip hash", ctx); + assert!(tree == hex::encode(tbytes(&self.tree)), "{} sapling tree differs from the reference", ctx); + assert_eq!(w.last_scanned_height(), tip, "{} last_scanned_height", ctx); + { + let wb = w.blocks.read().unwrap(); + assert_eq!(wb.len(), self.win.len(), "{} block window length", ctx); + assert_eq!(wb[0].height, *self.win.front().unwrap(), "{} block window start", ctx); + if full { + for (b, &hh) in wb.iter().zip(self.win.iter()) { + assert_eq!(b.height, hh, "{} window heights", ctx); + let rb = self.block_at(hh); + assert!(b.hash == rb.hash, "{} block hash at {}", ctx, hh); + assert!(tbytes(&b.tree) == tbytes(&rb.tree), "{} block tree at {}", ctx, hh); + } + } + } + + let txs = w.txs.read().unwrap(); + assert_eq!(txs.len(), self.rel.len(), "{} number of wallet txs", ctx); + let mut by_nf = HashMap::new(); + for (txid, wtx) in txs.iter() { + let r = self.rel.get(txid).unwrap_or_else(|| panic!("{} unexpected wallet tx {}", ctx, txid)); + assert_eq!(wtx.txid, *txid, "{} txid field", ctx); + assert_eq!(wtx.block, r.0, "{} block of tx {}", ctx, txid); + assert_eq!(wtx.total_shielded_value_spent, r.1, "{} total_shielded_value_spent of {}", ctx, txid); + for nd in wtx.notes.iter() { + assert!(by_nf.insert(nd.nullifier, (*txid, nd)).is_none(), "{} duplicate nullifier", ctx); + } + } + assert_eq!(by_nf.len(), self.notes.len(), "{} number of notes", ctx); + + let win_first = *self.win.front().unwrap(); + let window_full = self.win.len() > MAX_REORG; + let mut cleared = 0; + for (idx, n) in self.notes.iter().enumerate() { + let (txid, nd) = by_nf.get(&n.nf) + .unwrap_or_else(|| panic!("{} note at position {} missing (or wrong nullifier)", ctx, n.position)); + let nctx = format!("{} note#{} pos={} recv={} spent={:?} was_pending_at={:?}", ctx, idx, n.position, + n.recv, n.spent.map(|s| s.1), n.was_pending); + assert!(*txid == n.txid, "{} creating tx", nctx); + assert_eq!(nd.note.value, n.value, "{} value", nctx); + assert_eq!(nd.account, n.account, "{} account", nctx); + assert_eq!(nd.diversifier.0, n.to.diversifier.0, "{} diversifier", nctx); + assert_eq!(nd.is_change, n.is_change, "{} is_change", nctx); + assert_eq!(nd.spent, n.spent.map(|s| s.0), "{} spent", nctx); + assert_eq!(nd.unconfirmed_spent, n.pending, "{} unconfirmed_spent", nctx); + + let l = nd.witnesses.len(); + assert!(l <= MAX_REORG, "{} {} witnesses", nctx, l); + if let Some((_, spender_block)) = n.spent { + // A spent note keeps its legacy list, or (once the spend can no longer be + // rolled back) an empty one. + let cleared_ok = l == 0 && window_full && spender_block < win_first; + assert!(l == n.len || cleared_ok, "{} spent note has {} witnesses, expected {}", nctx, l, n.len); + if l != n.len { + cleared += 1; + } + continue; + } + assert_eq!(l, n.len, "{} witness list length", nctx); + if n.pending.is_some() || l == 0 { + continue; // a pending note's witness is not advanced (F5b) + } + assert!(wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), + "{} newest witness differs from the reference witness", nctx); + if full { + for i in 0..l { + let hh = tip - i as i32; + let rw = n.hist_at(hh).unwrap_or_else(|| panic!("{} no reference witness at {}", nctx, hh)); + assert!(wbytes(&nd.witnesses[l - 1 - i]) == wbytes(rw), + "{} witness for height {} differs from the reference", nctx, hh); + } + if idx % 5 == self.step % 5 || self.op == "end" { + self.check_semantics(n, &nd.witnesses[l - 1], &nctx); + for &i in &[cmp::min(4, l - 1), l - 1] { + assert!(nd.witnesses[l - 1 - i].root() == self.block_at(tip - i as i32).root, + "{} witness root for height {} is not that block's root", nctx, tip - i as i32); + } + } + } + } + if cleared > self.cleared_seen.get() { + self.cleared_seen.set(cleared); + } + + // Balances + let (_, off) = self.anchor(offset_cfg); + let (mut z, mut verified, mut spendable) = (0u64, 0u64, 0u64); + for n in self.notes.iter().filter(|n| n.live()) { + z += n.value; + if n.recv <= tip - off { + verified += n.value; + } + if n.recv <= tip - off - 1 && n.has_sk { + spendable += n.value; + } + } + drop(txs); + assert_eq!(w.zbalance(None), z, "{} zbalance", ctx); + assert_eq!(w.verified_zbalance(None), verified, "{} verified_zbalance", ctx); + assert_eq!(w.spendable_zbalance(None), spendable, "{} spendable_zbalance", ctx); + assert_eq!(w.unconfirmed_zbalance(None), 0, "{} unconfirmed_zbalance", ctx); + + let mp = w.mempool_txs.read().unwrap(); + let mine_ids: HashSet = mp.keys().cloned().collect(); + let want_ids: HashSet = if std::ptr::eq(w, &self.w) { + self.mempool.keys().cloned().collect() + } else { + HashSet::new() // a fresh copy + }; + assert!(mine_ids == want_ids, "{} mempool {:?} != {:?}", ctx, mine_ids, want_ids); + } + + fn check_semantics(&self, n: &RefNote, last: &IncrementalWitness, nctx: &str) { + let tip_root = self.block_at(self.tip()).root; + assert!(last.root() == tip_root, "{} newest witness root is not the tip root", nctx); + assert_eq!(last.position() as u64, n.position, "{} witness position", nctx); + let path = last.path().unwrap_or_else(|| panic!("{} no authentication path", nctx)); + assert_eq!(path.position, n.position, "{} path position", nctx); + assert!(path_root(&path, n.cmu) == tip_root, "{} the path does not lead from the cmu to the root", nctx); + } + + /// Per-address balances (slow: the wallet derives every note's address per call). + fn check_addresses(&self, w: &LightWallet) { + let ctx = self.ctx(); + let (tip, off) = self.anchor(self.cfg.anchor_offset); + let mut addrs: Vec = vec![]; + for n in self.notes.iter().filter(|n| n.live()) { + if !addrs.contains(&n.addr) && addrs.len() < 4 { + addrs.push(n.addr.clone()); + } + } + if let Some(a) = w.get_all_zaddresses().last() { + if !addrs.contains(a) { + addrs.push(a.clone()); + } + } + for a in addrs { + let live = self.notes.iter().filter(|n| n.live() && n.addr == a); + let (mut z, mut verified, mut spendable) = (0u64, 0u64, 0u64); + for n in live { + z += n.value; + if n.recv <= tip - off { + verified += n.value; + } + if n.recv <= tip - off - 1 && n.has_sk { + spendable += n.value; + } + } + assert_eq!(w.zbalance(Some(a.clone())), z, "{} zbalance({})", ctx, a); + assert_eq!(w.verified_zbalance(Some(a.clone())), verified, "{} verified_zbalance({})", ctx, a); + assert_eq!(w.spendable_zbalance(Some(a.clone())), spendable, "{} spendable_zbalance({})", ctx, a); + } + } + + fn summary(&self) -> String { + let s = &self.stats; + format!("seed {:#x}: {} blocks, {} outputs ({} ours, {} notes), {} compact spends, {} reorgs (max {}), \ + {} reloads, {} probes, {} sends ({} mined), {} insufficient, {} requeued/{} re-mined, \ + {} full checks, max {} cleared lists, {} keys; time gen {:.1}s scan {:.1}s ref {:.1}s \ + check {:.1}s send {:.1}s reorg {:.1}s reload {:.1}s periodic {:.1}s", + self.seed, s.blocks, s.outputs, s.ours, self.notes.len(), s.spends, s.reorgs, s.max_depth, + s.reloads, s.probes, s.sends, s.mined_sends, s.insufficient, s.requeued, s.remined, + s.full_checks, self.cleared_seen.get(), self.w.zkeys.read().unwrap().len(), + s.t_gen, s.t_scan, s.t_ref, s.t_check, s.t_send, s.t_reorg, s.t_reload, s.t_full) + } +} + +/// The most inputs any selection order can take for `target`: notes are taken while the running +/// total is below the target, and ascending order reaches it last. +fn worst_case_inputs(values: &[u64], target: u64) -> usize { + let mut v = values.to_vec(); + v.sort(); + let mut sum = 0u64; + for (j, x) in v.iter().enumerate() { + sum += x; + if sum >= target { + return j + 1; + } + } + v.len() +} + +fn compact_of(tx: &Transaction) -> CompactTx { + let mut ctx = CompactTx::new(); + ctx.set_hash(tx.txid().0.to_vec()); + for s in &tx.shielded_spends { + let mut cs = CompactSpend::new(); + cs.set_nf(s.nullifier.to_vec()); + ctx.spends.push(cs); + } + for o in &tx.shielded_outputs { + let mut cmu = vec![]; + o.cmu.into_repr().write_le(&mut cmu).unwrap(); + let mut epk = vec![]; + o.ephemeral_key.write(&mut epk).unwrap(); + let mut co = CompactOutput::new(); + co.set_cmu(cmu); + co.set_epk(epk); + co.set_ciphertext(o.enc_ciphertext[..52].to_vec()); + ctx.outputs.push(co); + } + ctx +} + +/// Which outputs of a wallet-built tx are ours: full trial decryption with each of our ivks. +fn owned_outputs(tx: &Transaction, keys: &[Key]) -> Vec> { + tx.shielded_outputs.iter().map(|o| { + let epk = match o.ephemeral_key.as_prime_order(&JUBJUB) { + Some(p) => p, + None => return None, + }; + keys.iter().enumerate().filter_map(|(account, k)| { + try_sapling_note_decryption(&k.fvk.fvk.vk.ivk(), &epk, &o.cmu, &o.enc_ciphertext) + .map(|(note, to, _)| Owned { account, note, to }) + }).next() + }).collect() +} + +// --------------------------------------------------------------------------------------------- +// Runs +// --------------------------------------------------------------------------------------------- + +#[derive(Clone, Copy, PartialEq)] +enum Proofs { + /// A probe at anchor offset 0, a probe at offset 4, and a real send that is mined (or, in + /// one run in five, left unmined near the end). + Three, + /// One proving send per seed: probe(0), probe(4) or a real send, by seed. + One, +} + +fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) { + assert!(n_blocks >= 20, "T6_BLOCKS must be >= 20"); + let t0 = Instant::now(); + let mut sim = Sim::new(seed); + sim.force_pay = 5; + sim.target_live = sim.rint(8, 40) as usize; + println!("oracle seed {:#x}: {} blocks, anchor_offset {}, start height {}, target live notes {}", seed, + n_blocks, sim.cfg.anchor_offset, sim.tip(), sim.target_live); + + let n = n_blocks; + let mut probe0 = Some(sim.rng.gen_range(n / 5, n / 2)); + let mut probe4 = Some(sim.rng.gen_range(n / 2, n * 4 / 5)); + let unmined = sim.rng.gen_bool(0.2) && n >= 60; + let mut send_at = Some(if unmined { n - 30 } else { sim.rng.gen_range(n / 4, n * 3 / 4) }); + match proofs { + Proofs::Three => {} + Proofs::One => match seed % 3 { + 0 => { probe4 = None; send_at = None; } + 1 => { probe0 = None; send_at = None; } + _ => { probe0 = None; probe4 = None; } + }, + } + let mut insufficient_at = Some(sim.rng.gen_range(0, n)); + let mut deep_reorg_at = if n > 150 && sim.rng.gen_bool(0.4) { Some(sim.rng.gen_range(120, n)) } else { None }; + let mut mine_at: Option = None; + + for step in 0..n { + sim.step = step; + let calm = sim.pending.is_none() && !sim.frozen; + + if calm { + let want = if deep_reorg_at.map_or(false, |s| step >= s) && sim.win.len() > 91 { + deep_reorg_at = None; + Some(cmp::min(sim.rint(90, 100) as usize, sim.win.len() - 1)) + } else if sim.rng.gen_range(0, 1000) < 30 && sim.win.len() > 1 { + Some(sim.pick_depth()) + } else { + None + }; + if let Some(k) = want.and_then(|k| sim.safe_depth(k)) { + let stepwise = sim.rng.gen_bool(0.3); + let t = Instant::now(); + sim.reorg(k, stepwise); + sim.stats.t_reorg += t.elapsed().as_secs_f64(); + } + if sim.rng.gen_range(0, 1000) < 15 { + let t = Instant::now(); + sim.reload(); + sim.stats.t_reload += t.elapsed().as_secs_f64(); + } + if probe0.map_or(false, |s| step >= s) { + probe0 = None; + sim.send(v, SendKind::Probe(0)); + } + if probe4.map_or(false, |s| step >= s) { + probe4 = None; + sim.send(v, SendKind::Probe(4)); + } + if send_at.map_or(false, |s| step >= s) { + send_at = None; + if sim.send(v, SendKind::Real) { + if unmined { + sim.frozen = true; + } else { + mine_at = Some(step + sim.rint(1, 12) as usize); + } + } + } + } + if insufficient_at.map_or(false, |s| step >= s) && sim.pending.is_none() { + insufficient_at = None; + sim.insufficient_check(); + } + + if mine_at.map_or(false, |s| step >= s) { + mine_at = None; + let p = sim.pending.take().unwrap(); + sim.mine(vec![p.gtx], true); + let mined_at = sim.tip(); + sim.pending_windows.push((p.sent_tip, mined_at)); + } else { + sim.mine(vec![], true); + } + if step % 50 == 49 { + sim.op = "periodic"; + sim.stats.full_checks += 1; + let t = Instant::now(); + sim.check(true); + sim.stats.t_full += t.elapsed().as_secs_f64(); + } + } + if let Some(p) = &sim.pending { + // An unmined send: by now it has expired from the mempool. + if sim.tip() >= *sim.mempool.get(&p.txid).unwrap_or(&i32::MIN) + TX_EXPIRY_DELTA { + assert!(sim.w.mempool_txs.read().unwrap().is_empty(), "{} expired send still in the mempool", + sim.ctx()); + } + } + sim.op = "end"; + sim.check(true); + sim.check_addresses(&sim.w); + println!("oracle {} ({:.1}s)", sim.summary(), t0.elapsed().as_secs_f64()); +} + +impl Sim { + /// Ask the wallet for one more than its eligible funds: must fail with the insufficient-funds + /// error, before any proving, and change nothing. + fn insufficient_check(&mut self) { + self.op = "insufficient"; + let (_, off) = self.anchor(self.cfg.anchor_offset); + let total: u64 = self.notes.iter() + .filter(|n| n.live() && n.has_sk && n.len >= off as usize + 1) + .map(|n| n.value).sum(); + let fee = u64::from(DEFAULT_FEE); + let amount = cmp::max((total + 1).saturating_sub(fee), 1); + let to = encode_payment_address(self.cfg.hrp_sapling_address(), &self.foreign[0].1); + let (ss, so) = (SaplingParams::get("sapling-spend.params").unwrap(), + SaplingParams::get("sapling-output.params").unwrap()); + let r = self.w.send_to_address(BRANCH_ID, &ss, &so, false, vec![(to.as_str(), amount, None)], &fee, + |_| -> Result { panic!("insufficient send broadcast") }); + match r { + Err(e) => assert!(e.starts_with("Insufficient verified funds"), "{} unexpected error {}", + self.ctx(), e), + Ok(_) => panic!("{} a send of {} with {} eligible succeeded", self.ctx(), amount, total), + } + self.stats.insufficient += 1; + self.check(false); + } +} + +fn env_u64(name: &str) -> Option { + std::env::var(name).ok().map(|s| { + let s = s.trim(); + if s.starts_with("0x") { + u64::from_str_radix(&s[2..], 16).expect("hex u64") + } else { + s.parse().expect("u64") + } + }) +} + +fn run_seeds(first: u64, count: u64, n_blocks: usize, proofs: Proofs) { + let v = Verifier::new(); + let n_blocks = env_u64("T6_BLOCKS").map_or(n_blocks, |b| b as usize); + let seeds: Vec = match env_u64("T6_SEED") { + Some(s) => vec![s], + None => (first..first + count).collect(), + }; + let t0 = Instant::now(); + for s in seeds.iter() { + run_seed(*s, n_blocks, proofs, &v); + } + println!("oracle: {} seeds from {:#x} in {:.1}s", seeds.len(), seeds[0], t0.elapsed().as_secs_f64()); +} + +#[test] +fn oracle_default_a() { + run_seeds(0x7_0000, 5, 300, Proofs::Three); +} + +#[test] +fn oracle_default_b() { + run_seeds(0x7_0100, 5, 300, Proofs::Three); +} + +#[test] +fn oracle_default_c() { + run_seeds(0x7_0200, 5, 300, Proofs::Three); +} + +#[test] +fn oracle_default_d() { + run_seeds(0x7_0300, 5, 300, Proofs::Three); +} + +#[test] +#[ignore] +fn oracle_full_0() { + run_seeds(0x1_0000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_1() { + run_seeds(0x1_1000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_2() { + run_seeds(0x1_2000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_3() { + run_seeds(0x1_3000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_4() { + run_seeds(0x1_4000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_5() { + run_seeds(0x1_5000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_6() { + run_seeds(0x1_6000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_full_7() { + run_seeds(0x1_7000, 125, 400, Proofs::One); +} + +#[test] +#[ignore] +fn oracle_soak_0() { + run_seeds(0x5_0000, 10, 5000, Proofs::Three); +} + +#[test] +#[ignore] +fn oracle_soak_1() { + run_seeds(0x5_1000, 10, 5000, Proofs::Three); +} + +/// T6_SEED= [T6_BLOCKS=]: replay one seed. +#[test] +#[ignore] +fn oracle_env_seed() { + let seed = env_u64("T6_SEED").expect("set T6_SEED"); + run_seeds(seed, 1, 300, Proofs::Three); +} + +/// F5a reproducer (pre-existing bug, out of Tier 6). A reorg while a send is pending panics in +/// invalidate_block: `nd.unconfirmed_spent.is_some() && txids_to_remove.contains(&nd.spent.unwrap())` +/// unwraps `spent`, which is None for a pending note. Fails on BASE; passes once fixed. +#[test] +#[ignore] +fn oracle_f5a_reorg_while_send_pending() { + let v = Verifier::new(); + let mut sim = Sim::new(0xF5A); + sim.force_pay = 12; + for _ in 0..12 { + sim.mine(vec![], true); + } + assert!(sim.send(&v, SendKind::Real), "setup: the send must be built"); + sim.mine(vec![], true); + sim.mine(vec![], true); + + let tip = sim.tip(); + let w = &sim.w; + let r = panic::catch_unwind(AssertUnwindSafe(|| w.invalidate_block(tip))); + assert!(r.is_ok(), "F5a (BASE bug): invalidate_block panicked while a send was pending \ + (it unwraps nd.spent of a note that only has unconfirmed_spent)"); + assert_eq!(r.unwrap(), 1); + sim.ref_invalidate(tip, 1); + sim.check(true); + + // The pending send can still be mined on the new branch. + let g = sim.pending.take().unwrap().gtx; + sim.mine(vec![g], true); + sim.check(true); +} + +/// F5b via a reorg (pre-existing bug, out of Tier 6; the F5b root cause). While a send is +/// pending, its notes' witness lists get a copy per block but are not advanced. Once the send is +/// mined the notes are spent and that does not matter, but a reorg that undoes the mining block +/// (to a height >= sent tip + 2) makes them spendable again with the stale entries of the pending +/// blocks, so a later send has an anchor that is not on chain. The green runs never reorg into +/// such a window (`Sim::safe_depth`). Fails on BASE; passes once pending notes stay current. +#[test] +#[ignore] +fn oracle_f5b_reorg_undoes_mined_send() { + let v = Verifier::new(); + let mut sim = Sim::new(0xF5C); + sim.force_pay = 12; + for _ in 0..12 { + sim.mine(vec![], true); + } + assert!(sim.send(&v, SendKind::Real), "setup: the send must be built"); + let sent_tip = sim.tip(); + sim.force_output = true; + for _ in 0..3 { + sim.mine(vec![], true); // pending: the chain moves, the pending note's witness does not + } + let p = sim.pending.take().unwrap(); + sim.mine(vec![p.gtx], true); + let mined_at = sim.tip(); + sim.mine(vec![], true); + + // A 2-block reorg undoes the mining block; the new branch does not re-mine the send. + let k = (sim.tip() - mined_at + 1) as usize; + assert_eq!(sim.w.invalidate_block(mined_at), k as u64); + sim.ref_invalidate(mined_at, k); + sim.requeue.clear(); + { + let txs = sim.w.txs.read().unwrap(); + for n in sim.notes.iter().filter(|n| n.live() && n.was_pending == Some(sent_tip)) { + let nd = txs.values().flat_map(|t| t.notes.iter()).find(|nd| nd.nullifier == n.nf).unwrap(); + let l = nd.witnesses.len(); + assert!(l > 0 && wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), + "F5b via reorg (BASE bug): after a reorg undid the block that mined our send, note at \ + position {} is spendable with the stale witness of its pending blocks", n.position); + } + } + sim.check(true); + sim.mine(vec![], true); +} + +/// F5b reproducer (pre-existing bug, out of Tier 6). A send that never mines leaves its notes +/// pending and their witnesses are no longer advanced. unconfirmed_spent is not saved, so after +/// a reload the note is spendable again with a stale witness, and a send that uses it has an +/// anchor that is not on chain. Fails on BASE; passes once pending notes stay current. +#[test] +#[ignore] +fn oracle_f5b_unmined_send_then_reload() { + let v = Verifier::new(); + let mut sim = Sim::new(0xF5B); + sim.force_pay = 12; + for _ in 0..12 { + sim.mine(vec![], true); + } + assert!(sim.send(&v, SendKind::Real), "setup: the send must be built"); + let p = sim.pending.take().unwrap(); // never mined + sim.force_output = true; + for _ in 0..(TX_EXPIRY_DELTA + 5) { + sim.mine(vec![], true); + } + assert!(sim.w.mempool_txs.read().unwrap().is_empty(), "the unmined send expired from mempool_txs"); + + sim.reload_raw(); + for n in sim.notes.iter_mut() { + if n.pending == Some(p.txid) { + n.pending = None; + } + } + { + let txs = sim.w.txs.read().unwrap(); + for n in sim.notes.iter().filter(|n| n.live()) { + let nd = txs.values().flat_map(|t| t.notes.iter()).find(|nd| nd.nullifier == n.nf).unwrap(); + let l = nd.witnesses.len(); + assert!(l > 0 && wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), + "F5b (BASE bug): after the unmined send and a reload, note at position {} is spendable \ + with a stale witness (its root is not the chain tip root)", n.position); + } + } + sim.check(true); +}