diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs index 3c86230..cec1bac 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet.rs @@ -2323,12 +2323,14 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 // Create a single mutable slice of all the newly-added witnesses. // A block without shielded outputs appends nothing to any witness: don't un-share (deep-clone) them. + // Advance every note whose nullifier is still tracked (spent.is_none(), the `nfs` set), pending + // ones included: a reload, or a reorg of the block that mined the send, makes them spendable again. let block_has_outputs = block.vtx.iter().any(|t| !t.outputs.is_empty()); let mut witness_refs: Vec<&mut IncrementalWitness> = if block_has_outputs { txs .values_mut() .map(|tx| tx.notes.iter_mut().filter_map( - |nd| if nd.spent.is_none() && nd.unconfirmed_spent.is_none() { nd.witnesses.last_mut().map(|w| Arc::make_mut(w)) } else { None })) + |nd| if nd.spent.is_none() { nd.witnesses.last_mut().map(|w| Arc::make_mut(w)) } else { None })) .flatten() .collect() } else { diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs index b2fbab2..5ccc766 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/oracle_tests.rs @@ -19,10 +19,10 @@ //! - witness-list lengths follow the legacy rule (one entry per block, at most 100, minus k on a //! k-block invalidate). A spent note may instead hold an empty list once its spender is below //! a full block window (the history can no longer be needed); -//! - every retained witness of an unspent, not-pending note equals (bytes) the reference witness -//! at that height, and sampled ones are checked semantically: root == reference root of that -//! block, position == the leaf index, and the authentication path hashes the note's cmu up to -//! the root; +//! - every retained witness of an unspent note (pending ones included) equals (bytes) the +//! reference witness at that height, and sampled ones are checked semantically: root == +//! reference root of that block, position == the leaf index, and the authentication path +//! hashes the note's cmu up to the root; //! - zbalance/verified/spendable balances (all, and per address at reload and at the end), the //! `scan_block` and `invalidate_block` return values, and the mempool (including expiry). //! @@ -33,22 +33,25 @@ //! signature. The first send of a run also checks that the verifier rejects a wrong anchor //! and a wrong nullifier. //! -//! Pending sends: reorgs of any depth happen while a send is pending, above its sent tip T, -//! across it (A < at <= T, A = its anchor height) or popping its anchor block (at <= A), in one -//! call or block by block. The wallet keeps `unconfirmed_spent` until a block spends the note or -//! a reload drops it (F5a, `oracle_f5a_*`). The chain checks an anchor root, not a height, so a -//! send whose anchor block was popped may still be mined: the reference orphans it (never mined, -//! its notes stay pending) when the reorg removed one of its notes, else with probability 1/2. -//! The fake chain does not validate anchors. A run that leaves a send unmined orphans it at once. +//! Pending sends (`oracle_f5a_*`, `oracle_f5b_*`). Inside the window of a real send the runs do, +//! at seeded rates: +//! - reorgs of any depth, above its sent tip T, across it (A < at <= T, A = its anchor height) or +//! popping its anchor block (at <= A), in one call or block by block. The wallet keeps +//! `unconfirmed_spent` until a block spends the note or a reload drops it (F5a). The chain +//! checks an anchor root, not a height, so a send whose anchor block was popped may still be +//! mined: the reference orphans it (never mined, its notes stay pending) when the reorg removed +//! one of its notes, else with probability 1/2. The fake chain does not validate anchors; +//! - reloads, which drop `unconfirmed_spent` (the send may still be mined afterwards; until it +//! is, the generator never spends its notes); +//! - probes on throwaway copies, which never know the flag, so they may spend a pending note; +//! - after the send is mined at M, reorgs with T + 2 <= at <= M that un-spend its notes. //! -//! Pre-existing bug the green runs avoid, with `#[ignore]` reproducers below: -//! - F5b: pending notes are not advanced, so their witnesses go stale. A reload drops -//! `unconfirmed_spent` and exposes such a note as spendable; so does a reorg that undoes the -//! block that mined the send back into the stale blocks. So no reload or throwaway copy happens -//! while a note is pending, and reorg depths are adjusted to stay out of that window. +//! A run that leaves a send unmined orphans it at once. Pending notes' witnesses are advanced +//! and compared like any unspent note's (F5b); a stale one fails the next check. //! //! Sizes: `oracle_default_*` (4 x 5 seeds x 300 blocks, three proving sends per seed); -//! `#[ignore]` `oracle_full_*` (8 x 125 seeds x 400 blocks, one proving send per seed) and +//! `#[ignore]` `oracle_full_*` (8 x 125 seeds x 400 blocks, one proving send per seed, plus +//! often a probe inside the pending window on seeds with a real send) and //! `oracle_soak_*` (2 x 10 seeds x 5000 blocks). Every run prints its seed. `T6_SEED=` //! (decimal or 0x-hex) replaces the seed list of any of them with that one seed, and //! `T6_BLOCKS=` overrides the block count; `oracle_env_seed` runs exactly that seed. @@ -342,9 +345,6 @@ impl RefNote { struct Pending { gtx: GenTx, txid: TxId, - /// The wallet's entries above this height may be stale (F5b): the sent tip, lowered to - /// at - 1 by a reorg at `at` while pending. - stale_from: i32, } #[derive(Clone, Copy, PartialEq)] @@ -378,6 +378,12 @@ struct Stats { orphaned: usize, /// Pending sends whose anchor block was popped but that are still mined. anchor_root_reuse: usize, + /// Reloads that dropped `unconfirmed_spent` of some note. + pending_reloads: usize, + /// Probes on a copy that spent a note pending in the wallet. + pending_probes: usize, + /// Reorgs that un-mined our send from a height >= sent tip + 2 (its notes become spendable). + unmine_reorgs: usize, t_gen: f64, t_scan: f64, t_ref: f64, @@ -413,9 +419,6 @@ struct Sim { last_send: Option<(TxId, i32, i32)>, /// Overrides the coin flip that orphans a pending send whose anchor block was popped. force_orphan: Option, - /// (stale_from, block that first mined it) of our mined sends: a reorg to a height in - /// (stale_from + 1, mined] would revive their notes with stale witnesses (F5b via a reorg). - pending_windows: Vec<(i32, i32)>, /// Blocks still to be paid to us (setup helper). force_pay: usize, force_output: bool, @@ -498,7 +501,6 @@ impl Sim { pending: None, last_send: None, force_orphan: None, - pending_windows: vec![], force_pay: 0, force_output: false, controls_done: false, @@ -683,8 +685,15 @@ impl Sim { GenTx { ctx, ours, raw: None, anchor_height: None } } + /// Nullifiers of the send in flight. After a reload the wallet no longer marks its notes + /// pending, but the send may still be mined, so nothing else may spend them. + fn reserved(&self) -> Vec<[u8; 32]> { + self.pending.as_ref().map_or(vec![], |p| p.gtx.nfs()) + } + fn gen_txs(&mut self, h: i32, keys: &[Key], already: &[GenTx]) -> Vec { let mut used: HashSet<[u8; 32]> = already.iter().flat_map(|g| g.nfs()).collect(); + used.extend(self.reserved()); let mut out = vec![]; if self.force_pay > 0 { self.force_pay -= 1; @@ -721,7 +730,8 @@ impl Sim { /// A requeued tx may only be re-mined if it does not double-spend. fn still_valid(&self, g: &GenTx, in_block: &[GenTx]) -> bool { - let used: HashSet<[u8; 32]> = in_block.iter().flat_map(|x| x.nfs()).collect(); + let mut used: HashSet<[u8; 32]> = in_block.iter().flat_map(|x| x.nfs()).collect(); + used.extend(self.reserved()); g.nfs().iter().all(|nf| { !used.contains(nf) && self.notes.iter().find(|n| n.nf == *nf).map_or(true, |n| n.live()) }) @@ -735,9 +745,11 @@ impl Sim { let mut txs: Vec = vec![]; let mut later = vec![]; + // `extra` (our send, once a reload dropped its pending flags) must not be double-spent either. + let extra_nfs: HashSet<[u8; 32]> = extra.iter().flat_map(|g| g.nfs()).collect(); for (at, g) in std::mem::replace(&mut self.requeue, vec![]) { if at <= h { - if self.still_valid(&g, &txs) { + if self.still_valid(&g, &txs) && g.nfs().iter().all(|nf| !extra_nfs.contains(nf)) { txs.push(g); self.stats.remined += 1; } @@ -797,8 +809,8 @@ impl Sim { /// Apply a block to the reference. Returns what scan_block must return. fn ref_apply(&mut self, h: i32, hash: BlockHash, txs: &[GenTx], keys: &[Key]) -> Vec { // The wallet tracks the nullifiers of every note not confirmed spent at block start - // (pending ones included). The reference advances the true witness of that same set - // (the wallet skips pending notes, F5b). + // (pending ones included) and advances the witnesses of that same set; so does the + // reference. let tracked: HashMap<[u8; 32], usize> = (0..self.notes.len()) .filter(|&i| self.notes[i].spent.is_none()) .map(|i| (self.notes[i].nf, i)) @@ -895,7 +907,6 @@ impl Sim { self.win.pop_front(); } let win_first = *self.win.front().unwrap(); - self.pending_windows.retain(|(_, m)| *m >= win_first); for n in self.notes.iter_mut() { while n.hist.front().map_or(false, |(hh, _)| *hh < h - KEEP) { n.hist.pop_front(); @@ -929,11 +940,14 @@ impl Sim { assert!(k >= 1 && k < self.win.len(), "{} bad reorg depth {}", self.ctx(), k); self.op = "reorg"; let at = self.tip() - k as i32 + 1; - if let Some((_, t, a)) = self.last_send { + if let Some((txid, t, a)) = self.last_send { if self.any_pending() { let class = if at > t { 0 } else if at > a { 1 } else { 2 }; self.stats.pending_reorgs[class] += 1; } + if at >= t + 2 && self.rel.get(&txid).map_or(false, |r| r.0 >= at) { + self.stats.unmine_reorgs += 1; + } } let n = if stepwise { (0..k).map(|_| self.w.invalidate_block(self.w.last_scanned_height())).sum::() @@ -979,9 +993,6 @@ impl Sim { self.stats.anchor_root_reuse += 1; } } - if let Some(p) = self.pending.as_mut() { - p.stale_from = cmp::min(p.stale_from, at - 1); - } self.notes.retain(|n| n.recv < at); let ctx = self.ctx(); @@ -1000,7 +1011,6 @@ impl Sim { } } self.rel.retain(|_, r| r.0 < at); - self.pending_windows.retain(|(_, m)| *m < at); // The new branch may re-mine the dropped spends at other heights, or drop them. let mut dropped = vec![]; @@ -1031,25 +1041,6 @@ impl Sim { cmp::min(d as usize, max) } - /// Adjust a reorg depth so the reorg does not undo the mining of one of our sends back into - /// the blocks where it was pending (F5b via a reorg, see its #[ignore] reproducer): prefer - /// keeping the mining block, else pop the whole pending window. - fn safe_depth(&self, k: usize) -> Option { - let tip = self.tip(); - let max = self.win.len() as i32 - 1; - let mut at = tip - k as i32 + 1; - for _ in 0..8 { - match self.pending_windows.iter().find(|(t, m)| *t + 1 < at && at <= *m) { - None => { - let k = tip - at + 1; - return if k >= 1 && k <= max { Some(k as usize) } else { None }; - } - Some(&(t, m)) => at = if m + 1 <= tip { m + 1 } else { t + 1 }, - } - } - None - } - /// Some note is pending: a send is in flight, or an orphaned one locked its notes until reload. fn any_pending(&self) -> bool { self.notes.iter().any(|n| n.pending.is_some()) @@ -1090,6 +1081,21 @@ impl Sim { Some((tip - at + 1) as usize) } + /// A reorg depth that un-mines our last send (mined at M) back to a height in [T + 2, M]: + /// its notes become spendable again with the witnesses of the blocks where they were pending. + fn unmine_reorg_depth(&mut self) -> Option { + let (txid, t, _) = self.last_send?; + let m = self.rel.get(&txid)?.0; + let tip = self.tip(); + let lo = cmp::max(t + 2, *self.win.front().unwrap() + 1); + let hi = cmp::min(m, tip); + if lo > hi { + return None; + } + let at = self.rng.gen_range(lo, hi + 1); + Some((tip - at + 1) as usize) + } + /// Orphan the pending send: it is never mined, and its notes stay pending until a reload. fn orphan_pending(&mut self) { if self.pending.take().is_some() { @@ -1098,7 +1104,6 @@ impl Sim { } fn reload(&mut self) { - assert!(!self.any_pending(), "{} reload with a pending note (F5b)", self.ctx()); self.op = "reload"; self.reload_raw(); self.stats.reloads += 1; @@ -1120,31 +1125,42 @@ impl Sim { w3.write(&mut b3).unwrap(); assert!(b2 == b3, "{} write(read(x)) is not idempotent", self.ctx()); self.w = w3; - self.mempool.clear(); // mempool_txs is not saved (an orphaned send may still be in it) + // Neither unconfirmed_spent nor mempool_txs is saved. A send still in flight may be mined + // later (the generator leaves its notes alone, see `reserved`). + if self.any_pending() { + self.stats.pending_reloads += 1; + } + for n in self.notes.iter_mut() { + n.pending = None; + } + self.mempool.clear(); if self.win.len() > MAX_REORG { self.clear_bound = cmp::max(self.clear_bound, *self.win.front().unwrap()); } } - /// Build a send. Probe: on a throwaway copy with the given anchor offset. Real: on the - /// wallet itself, which leaves it pending. Returns whether a proving send was made (false - /// when there were not enough eligible funds; the error path is checked instead). + /// Build a send. Probe: on a throwaway copy with the given anchor offset (the copy does not + /// know which notes are pending, so it may spend them). Real: on the wallet itself, which + /// leaves it pending. Returns whether a proving send was made (false when there were not + /// enough eligible funds; the error path is checked instead). fn send(&mut self, v: &Verifier, kind: SendKind) -> bool { - self.send_with(v, kind, None) + self.send_with(v, kind, None).is_some() } - /// `send` with an optional fixed amount (one recipient). With a fixed amount the spent notes - /// must be exactly the ones the wallet's selection rule picks from the eligible notes; it - /// returns false without sending when they do not cover it or need more than 10 inputs. - fn send_with(&mut self, v: &Verifier, kind: SendKind, fixed: Option) -> bool { + /// `send` with an optional fixed amount (one recipient); returns the spent nullifiers. With a + /// fixed amount they must be exactly the ones the wallet's selection rule picks from the + /// eligible notes; it returns None without sending when those do not cover the amount or + /// need more than 10 inputs. + fn send_with(&mut self, v: &Verifier, kind: SendKind, fixed: Option) -> Option> { assert!(kind != SendKind::Real || self.pending.is_none(), "{} send with a pending send", self.ctx()); self.op = match kind { SendKind::Probe(_) => "probe-send", SendKind::Real => "send" }; let offset_cfg = match kind { SendKind::Probe(o) => o, SendKind::Real => self.cfg.anchor_offset }; let (tip, off) = self.anchor(offset_cfg); + let fresh = kind != SendKind::Real; let elig: Vec = (0..self.notes.len()) .filter(|&i| { let n = &self.notes[i]; - n.live() && n.has_sk && n.len >= off as usize + 1 + n.spent.is_none() && (fresh || n.pending.is_none()) && n.has_sk && n.len >= off as usize + 1 }) .collect(); let total: u64 = elig.iter().map(|&i| self.notes[i].value).sum(); @@ -1176,14 +1192,14 @@ impl Sim { .map(|&i| (self.notes[i].value, self.notes[i].nf)).collect(); match expected_selection(&cands, a + fee) { Some(s) if s.len() <= 10 => Some(s.into_iter().collect()), - _ => return false, + _ => return None, } } }; if covering.is_empty() && fixed.is_none() { // Not enough: check the error path instead. self.insufficient_check(); - return false; + return None; } let (ss, so) = (SaplingParams::get("sapling-spend.params").unwrap(), SaplingParams::get("sapling-output.params").unwrap()); @@ -1282,6 +1298,9 @@ impl Sim { match kind { SendKind::Probe(_) => { self.stats.probes += 1; + if self.notes.iter().any(|n| n.pending.is_some() && tx_nfs.contains(&n.nf)) { + self.stats.pending_probes += 1; + } } SendKind::Real => { for n in self.notes.iter_mut() { @@ -1297,13 +1316,13 @@ impl Sim { raw: Some(raw), anchor_height: Some(tip - off), }; - self.pending = Some(Pending { gtx, txid, stale_from: tip }); + self.pending = Some(Pending { gtx, txid }); self.last_send = Some((txid, tip, tip - off)); self.stats.sends += 1; self.check(false); } } - true + Some(tx_nfs) } fn check(&self, full: bool) { @@ -1314,6 +1333,8 @@ impl Sim { fn check_wallet(&self, w: &LightWallet, offset_cfg: u32, full: bool) { let ctx = self.ctx(); let tip = self.tip(); + // A fresh copy (read of a written file) knows neither pending notes nor the mempool. + let fresh = !std::ptr::eq(w, &self.w); let (h, hash, tree) = w.get_sapling_tree().unwrap(); assert_eq!(h, tip, "{} sapling tree height", ctx); @@ -1363,7 +1384,7 @@ impl Sim { assert_eq!(nd.diversifier.0, n.to.diversifier.0, "{} diversifier", nctx); assert_eq!(nd.is_change, n.is_change, "{} is_change", nctx); assert_eq!(nd.spent, n.spent.map(|s| s.0), "{} spent", nctx); - assert_eq!(nd.unconfirmed_spent, n.pending, "{} unconfirmed_spent", nctx); + assert_eq!(nd.unconfirmed_spent, if fresh { None } else { n.pending }, "{} unconfirmed_spent", nctx); let l = nd.witnesses.len(); assert!(l <= MAX_REORG, "{} {} witnesses", nctx, l); @@ -1378,8 +1399,8 @@ impl Sim { continue; } assert_eq!(l, n.len, "{} witness list length", nctx); - if n.pending.is_some() || l == 0 { - continue; // a pending note's witness is not advanced (F5b) + if l == 0 { + continue; } assert!(wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), "{} newest witness differs from the reference witness", nctx); @@ -1406,7 +1427,7 @@ impl Sim { // Balances let (_, off) = self.anchor(offset_cfg); let (mut z, mut verified, mut spendable) = (0u64, 0u64, 0u64); - for n in self.notes.iter().filter(|n| n.live()) { + for n in self.notes.iter().filter(|n| n.spent.is_none() && (fresh || n.pending.is_none())) { z += n.value; if n.recv <= tip - off { verified += n.value; @@ -1423,10 +1444,10 @@ impl Sim { let mp = w.mempool_txs.read().unwrap(); let mine_ids: HashSet = mp.keys().cloned().collect(); - let want_ids: HashSet = if std::ptr::eq(w, &self.w) { - self.mempool.keys().cloned().collect() + let want_ids: HashSet = if fresh { + HashSet::new() } else { - HashSet::new() // a fresh copy + self.mempool.keys().cloned().collect() }; assert!(mine_ids == want_ids, "{} mempool {:?} != {:?}", ctx, mine_ids, want_ids); } @@ -1489,9 +1510,10 @@ impl Sim { impl Stats { fn pending_summary(&self) -> String { - format!("pending: reorgs above/across/popped {}/{}/{}, orphaned {}, anchor-root reuse {}", + format!("pending: reorgs above/across/popped {}/{}/{}, orphaned {}, anchor-root reuse {}, reloads {}, \ + probes {}, un-mining reorgs {}", self.pending_reorgs[0], self.pending_reorgs[1], self.pending_reorgs[2], self.orphaned, - self.anchor_root_reuse) + self.anchor_root_reuse, self.pending_reloads, self.pending_probes, self.unmine_reorgs) } /// Add the pending-state counters of one seed. @@ -1501,12 +1523,15 @@ impl Stats { } self.orphaned += o.orphaned; self.anchor_root_reuse += o.anchor_root_reuse; + self.pending_reloads += o.pending_reloads; + self.pending_probes += o.pending_probes; + self.unmine_reorgs += o.unmine_reorgs; } /// Every pending-state class was hit (asserted by the full runs). fn assert_pending_cover(&self) { let all = [self.pending_reorgs[0], self.pending_reorgs[1], self.pending_reorgs[2], self.orphaned, - self.anchor_root_reuse]; + self.anchor_root_reuse, self.pending_reloads, self.pending_probes, self.unmine_reorgs]; assert!(all.iter().all(|&c| c > 0), "a pending-state class was never hit: {}", self.pending_summary()); } } @@ -1623,12 +1648,17 @@ fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) -> Stats { _ => { probe0 = None; probe4 = None; } }, } - // Rate of a reorg inside the pending window of the real send. - let p_pending_reorg = match proofs { Proofs::Three => 0.4, Proofs::One => 0.9 }; + // Rates, per real send, of (a) a reorg, (b) a reload and (c) a probe inside its pending window, + // and (d) a reorg that un-mines it again. + let (p_reorg, p_reload, p_probe, p_unmine) = match proofs { + Proofs::Three => (0.4, 0.3, 0.3, 0.4), + Proofs::One => (0.9, 0.5, 0.7, 0.8), + }; let mut insufficient_at = Some(sim.rng.gen_range(0, n)); let mut deep_reorg_at = if n > 150 && sim.rng.gen_bool(0.4) { Some(sim.rng.gen_range(120, n)) } else { None }; let mut mine_at: Option = None; - let mut pending_reorg_at: Option = None; + let (mut pending_reorg_at, mut pending_reload_at, mut pending_probe_at, mut unmine_at): (Option, + Option, Option, Option) = (None, None, None, None); for step in 0..n { sim.step = step; @@ -1636,6 +1666,9 @@ fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) -> Stats { let want = if pending_reorg_at == Some(step) { pending_reorg_at = None; sim.pending_reorg_depth() + } else if unmine_at == Some(step) { + unmine_at = None; + sim.unmine_reorg_depth() } else if deep_reorg_at.map_or(false, |s| step >= s) && sim.win.len() > 91 { deep_reorg_at = None; Some(cmp::min(sim.rint(90, 100) as usize, sim.win.len() - 1)) @@ -1644,23 +1677,33 @@ fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) -> Stats { } else { None }; - if let Some(k) = want.and_then(|k| sim.safe_depth(k)) { + if let Some(k) = want { let stepwise = sim.rng.gen_bool(0.3); let t = Instant::now(); sim.reorg(k, stepwise); sim.stats.t_reorg += t.elapsed().as_secs_f64(); } - // F5b: no reload or throwaway copy while a note is pending. - if sim.rng.gen_range(0, 1000) < 15 && !sim.any_pending() { + if sim.rng.gen_range(0, 1000) < 15 || pending_reload_at == Some(step) { let t = Instant::now(); sim.reload(); sim.stats.t_reload += t.elapsed().as_secs_f64(); } - if probe0.map_or(false, |s| step >= s) && !sim.any_pending() { + if pending_probe_at == Some(step) { + // Spend a pending note on a copy: the amount that the note alone covers exactly. + let fee = u64::from(DEFAULT_FEE); + let vals: Vec = sim.notes.iter().filter(|n| n.pending.is_some() && n.value > fee) + .map(|n| n.value).collect(); + if !vals.is_empty() { + let vp = vals[sim.rng.gen_range(0, vals.len())]; + let off = if sim.rng.gen_bool(0.5) { 0 } else { 4 }; + sim.send_with(v, SendKind::Probe(off), Some(vp - fee)); + } + } + if probe0.map_or(false, |s| step >= s) { probe0 = None; sim.send(v, SendKind::Probe(0)); } - if probe4.map_or(false, |s| step >= s) && !sim.any_pending() { + if probe4.map_or(false, |s| step >= s) { probe4 = None; sim.send(v, SendKind::Probe(4)); } @@ -1673,8 +1716,15 @@ fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) -> Stats { } else { mine_at = Some(end); } - if sim.rng.gen_bool(p_pending_reorg) { - pending_reorg_at = Some(step + sim.rint(1, (end - step) as u64) as usize); + let span = (end - step) as u64; + if sim.rng.gen_bool(p_reorg) { + pending_reorg_at = Some(step + sim.rint(1, span) as usize); + } + if sim.rng.gen_bool(p_reload) { + pending_reload_at = Some(step + sim.rint(1, span) as usize); + } + if sim.rng.gen_bool(p_probe) { + pending_probe_at = Some(step + sim.rint(1, span) as usize); } } } @@ -1688,8 +1738,9 @@ fn run_seed(seed: u64, n_blocks: usize, proofs: Proofs, v: &Verifier) -> Stats { // A reorg may have orphaned it. if let Some(p) = sim.pending.take() { sim.mine(vec![p.gtx], true); - let mined_at = sim.tip(); - sim.pending_windows.push((p.stale_from, mined_at)); + if sim.rng.gen_bool(p_unmine) { + unmine_at = Some(step + sim.rint(1, 6) as usize); + } } else { sim.mine(vec![], true); } @@ -1919,7 +1970,7 @@ fn oracle_f5a_monitor_entry_reorg_keeps_pending() { let fee = u64::from(DEFAULT_FEE); let vq = sim.notes.iter().filter(|n| n.live() && n.has_sk && n.len >= off as usize + 1 && n.value > fee + 1) .map(|n| n.value).min().expect("setup: a note to spend"); - assert!(sim.send_with(&v, SendKind::Real, Some(vq - fee - 1)), "setup: the send must be built"); + assert!(sim.send_with(&v, SendKind::Real, Some(vq - fee - 1)).is_some(), "setup: the send must be built"); let t = sim.tip(); let (txid, a) = { let p = sim.pending.as_ref().unwrap(); @@ -1958,14 +2009,11 @@ fn oracle_f5a_monitor_entry_reorg_keeps_pending() { } } -/// F5b via a reorg (pre-existing bug, out of Tier 6; the F5b root cause). While a send is -/// pending, its notes' witness lists get a copy per block but are not advanced. Once the send is -/// mined the notes are spent and that does not matter, but a reorg that undoes the mining block -/// (to a height >= sent tip + 2) makes them spendable again with the stale entries of the pending -/// blocks, so a later send has an anchor that is not on chain. The green runs never reorg into -/// such a window (`Sim::safe_depth`). Fails on BASE; passes once pending notes stay current. +/// F5b via a reorg (regression test). While a send was pending, its notes' witness lists got a +/// copy per block but were not advanced. Once the send is mined the notes are spent and that does +/// not matter, but a reorg that undoes the mining block (to a height >= sent tip + 2) makes them +/// spendable again with the entries of the pending blocks, which must be the true witnesses. #[test] -#[ignore] fn oracle_f5b_reorg_undoes_mined_send() { let v = Verifier::new(); let mut sim = Sim::new(0xF5C); @@ -1995,7 +2043,7 @@ fn oracle_f5b_reorg_undoes_mined_send() { let nd = txs.values().flat_map(|t| t.notes.iter()).find(|nd| nd.nullifier == n.nf).unwrap(); let l = nd.witnesses.len(); assert!(l > 0 && wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), - "F5b via reorg (BASE bug): after a reorg undid the block that mined our send, note at \ + "F5b via reorg (regression): after a reorg undid the block that mined our send, note at \ position {} is spendable with the stale witness of its pending blocks", n.position); } } @@ -2003,12 +2051,11 @@ fn oracle_f5b_reorg_undoes_mined_send() { sim.mine(vec![], true); } -/// F5b reproducer (pre-existing bug, out of Tier 6). A send that never mines leaves its notes -/// pending and their witnesses are no longer advanced. unconfirmed_spent is not saved, so after -/// a reload the note is spendable again with a stale witness, and a send that uses it has an -/// anchor that is not on chain. Fails on BASE; passes once pending notes stay current. +/// F5b (regression test). A send that never mines leaves its notes pending. unconfirmed_spent is +/// not saved, so after a reload the notes are spendable again, and their witnesses must have +/// been advanced while they were pending (else a send that uses one has an anchor that is not on +/// chain). #[test] -#[ignore] fn oracle_f5b_unmined_send_then_reload() { let v = Verifier::new(); let mut sim = Sim::new(0xF5B); @@ -2017,28 +2064,99 @@ fn oracle_f5b_unmined_send_then_reload() { sim.mine(vec![], true); } assert!(sim.send(&v, SendKind::Real), "setup: the send must be built"); - let p = sim.pending.take().unwrap(); // never mined + sim.orphan_pending(); // never mined sim.force_output = true; for _ in 0..(TX_EXPIRY_DELTA + 5) { sim.mine(vec![], true); } assert!(sim.w.mempool_txs.read().unwrap().is_empty(), "the unmined send expired from mempool_txs"); - sim.reload_raw(); - for n in sim.notes.iter_mut() { - if n.pending == Some(p.txid) { - n.pending = None; - } - } + sim.reload_raw(); // drops unconfirmed_spent, in the wallet and in the reference { let txs = sim.w.txs.read().unwrap(); for n in sim.notes.iter().filter(|n| n.live()) { let nd = txs.values().flat_map(|t| t.notes.iter()).find(|nd| nd.nullifier == n.nf).unwrap(); let l = nd.witnesses.len(); assert!(l > 0 && wbytes(&nd.witnesses[l - 1]) == wbytes(&n.witness), - "F5b (BASE bug): after the unmined send and a reload, note at position {} is spendable \ + "F5b (regression): after the unmined send and a reload, note at position {} is spendable \ with a stale witness (its root is not the chain tip root)", n.position); } } sim.check(true); } + +/// F5b: a probe on a throwaway copy (which never knows the pending flags) spends a note that is +/// pending in the wallet, at anchor offsets 0 and 4. Both anchors lie in the blocks mined while +/// the note was pending, so its witnesses there must be the true ones. +#[test] +fn oracle_f5b_probe_while_pending() { + let v = Verifier::new(); + let mut sim = Sim::new(0xF5F); + sim.force_pay = 12; + for _ in 0..12 { + sim.mine(vec![], true); + } + assert!(sim.send(&v, SendKind::Real), "setup: the send must be built"); + sim.force_output = true; + for _ in 0..6 { + sim.mine(vec![], true); + } + let fee = u64::from(DEFAULT_FEE); + let (nf, vp) = sim.notes.iter().filter(|n| n.pending.is_some() && n.value > fee).map(|n| (n.nf, n.value)) + .next().expect("setup: a pending note above the fee"); + for &off in &[0u32, 4] { + let spent = sim.send_with(&v, SendKind::Probe(off), Some(vp - fee)) + .unwrap_or_else(|| panic!("setup: the probe at offset {} must be built", off)); + let want: HashSet<[u8; 32]> = [nf].iter().cloned().collect(); + assert!(spent == want, "the probe at offset {} did not spend exactly the pending note", off); + } + assert!(sim.stats.pending_probes == 2); + sim.mine(vec![], true); +} + +/// F5a + F5b: a reorg pops the anchor block of a pending send, which is orphaned; its note q +/// (received below the anchor) stays pending in the wallet. After a reload q is spendable again +/// and a new send of exactly q must verify: q's witnesses were advanced while it was pending. +#[test] +fn oracle_f5a_reorg_pops_anchor_then_reload_send() { + let v = Verifier::new(); + let mut sim = Sim::new(0xF5E); + sim.force_pay = 12; + for _ in 0..12 { + sim.mine(vec![], true); + } + for _ in 0..2 { + sim.mine(vec![], true); + } + let (_, off) = sim.anchor(sim.cfg.anchor_offset); + let a = sim.tip() - off; + let fee = u64::from(DEFAULT_FEE); + let (q, vq) = sim.notes.iter() + .filter(|n| n.live() && n.has_sk && n.len >= off as usize + 1 && n.value > fee && n.recv < a) + .map(|n| (n.nf, n.value)).max_by_key(|x| x.1).expect("setup: a note below the anchor"); + let want: HashSet<[u8; 32]> = [q].iter().cloned().collect(); + let spent = sim.send_with(&v, SendKind::Real, Some(vq - fee)).expect("setup: the first send must be built"); + assert!(spent == want, "setup: the first send spends exactly q"); + sim.force_output = true; + for _ in 0..3 { + sim.mine(vec![], true); + } + + // Reorg to the anchor height: the anchor block is popped and the send orphaned. + sim.force_orphan = Some(true); + let k = (sim.tip() - a + 1) as usize; + sim.reorg(k, false); + assert!(sim.pending.is_none(), "the send was orphaned"); + { + let txs = sim.w.txs.read().unwrap(); + let nd = txs.values().flat_map(|t| t.notes.iter()).find(|nd| nd.nullifier == q).expect("q kept"); + assert!(nd.unconfirmed_spent.is_some(), "q stays pending across the reorg"); + } + for _ in 0..2 { + sim.mine(vec![], true); + } + sim.reload(); + let spent = sim.send_with(&v, SendKind::Real, Some(vq - fee)).expect("the second send must be built"); + assert!(spent == want, "the second send spends exactly the formerly pending note q"); + sim.mine(vec![], true); +} diff --git a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/witness_tests.rs b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/witness_tests.rs index 668caf6..cb2c583 100644 --- a/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/witness_tests.rs +++ b/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/lightwallet/witness_tests.rs @@ -424,8 +424,9 @@ impl Chain { } } -/// T0: for every live note (L entries, B retained blocks) assert L <= 100, L <= B and, for k in -/// {0, 1, L-1}, witnesses[L-1-k].root() == blocks[B-1-k].tree.root(): the anchor at offset k. +/// T0: for every unspent note, pending ones included (L entries, B retained blocks) assert L <= 100, +/// L <= B and, for k in {0, 1, L-1}, witnesses[L-1-k].root() == blocks[B-1-k].tree.root(): the +/// anchor at offset k. fn check_anchor_alignment(w: &LightWallet, ctx: &str) -> usize { let blocks = w.blocks.read().unwrap(); let b = blocks.len(); @@ -433,7 +434,7 @@ fn check_anchor_alignment(w: &LightWallet, ctx: &str) -> usize { let mut checked = 0; for tx in txs.values() { for nd in tx.notes.iter() { - if nd.spent.is_some() || nd.unconfirmed_spent.is_some() { + if nd.spent.is_some() { continue; } let l = nd.witnesses.len();