Files
ObsidianDragon/third_party/silentdragonxlite/silentdragonxlite-cli/lib/src/grpcconnector.rs
T
DanSandClaude Opus 5.5 c233d4d503 perf(lite-backend): fetch full transactions over one shared gRPC channel
At the end of a sync the wallet fetches the full transaction of each new
wallet tx, together with every other transaction in the blocks that
contain them, as decoys. Every txid got its own pool job that built a
fresh Tokio runtime (one thread per CPU), a new rustls config and a new
TCP + TLS + HTTP/2 connection for a single GetTransaction call, so a
restore or rescan opened one connection per fetched transaction. The
first error returned from the sync while the remaining jobs kept
fetching and scanning in the background, and a stalled connection, an
unparsable transaction or a panicking scan could hang the sync forever
with its lock held.

fulltx_fetch.rs now works through the same (txid, height)-sorted,
de-duplicated list on the sync's runtime, with as many workers as the
sync pool has threads (2-8, the same fan-out as before), over one gRPC
channel that is dialed lazily and re-dialed on retry. Parsing and
scanning stay on the sync pool, and the call returns once the whole
batch has finished.

Behaviour changes:
- a failure on a decoy no longer fails the sync; it fails only when one
  of the wallet's own txs could not be fetched and scanned;
- new error texts "Error reading transaction: ...", "full tx scan job
  panicked" and "fetched only X of Y wallet txs", all after the existing
  "Error fetching all txns for memos: " prefix;
- each tx gets up to 3 fetch attempts with a 120 s timeout each, where
  the old code could wait forever; a request that legitimately takes
  longer now fails;
- nothing keeps running in the background after the sync returns;
- all GetTransaction calls share one TLS/HTTP-2 connection (up to one
  dial per worker at the start, plus a re-dial per retry); the request
  list and its order are unchanged;
- if the network silently drops packets mid-batch, every attempt waits
  out its timeout until 16 consecutive failed txs stop the batch: up to
  about 12 minutes with 8 workers (48 minutes with 2) with the sync lock
  held, where the old code gave up at its first failed connection.

Which txs are requested, in what order, how often they are retried and
when the batch stops do not depend on which txs are the wallet's own;
only the final result does. A server that keeps failing one tx can see
that result through the sync's retry, but it could already tell, because
an own tx that failed is requested again on the next sync without its
decoys; nothing new leaks.

tests/fulltx_fetch.rs drives the engine with a fake fetcher: the old
request list, each item fetched and scanned once, bounded concurrency, a
request sequence independent of ownership, retries, timeouts, the
breaker, panicking and unparsable scans, and a saved wallet
byte-identical to the old serial fill. The gRPC fetcher itself is not
covered by these tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 09:50:03 -05:00

407 lines
14 KiB
Rust

// Copyright The Hush Developers 2019-2022
// Released under the GPLv3
use log::{info,error};
use std::sync::Arc;
use zcash_primitives::transaction::{TxId};
use crate::grpc_client::{ChainSpec, BlockId, BlockRange, RawTransaction, CompactBlock,
TransparentAddressBlockFilter, TxFilter, Empty, LightdInfo, Coinsupply};
use tonic::transport::{Channel, ClientTlsConfig};
use tokio_rustls::{rustls::ClientConfig};
use tonic::{Request};
use threadpool::ThreadPool;
use std::sync::mpsc::channel;
use crate::PubCertificate;
use crate::grpc_client::compact_tx_streamer_client::CompactTxStreamerClient;
mod danger {
use tokio_rustls::rustls;
use webpki;
pub struct NoCertificateVerification {}
impl rustls::ServerCertVerifier for NoCertificateVerification {
fn verify_server_cert(&self,
_roots: &rustls::RootCertStore,
_presented_certs: &[rustls::Certificate],
_dns_name: webpki::DNSNameRef<'_>,
_ocsp: &[u8]) -> Result<rustls::ServerCertVerified, rustls::TLSError> {
Ok(rustls::ServerCertVerified::assertion())
}
}
}
async fn get_client(uri: &http::Uri, no_cert: bool) -> Result<CompactTxStreamerClient<Channel>, Box<dyn std::error::Error>> {
let channel = if uri.scheme_str() == Some("http") {
Channel::builder(uri.clone()).connect().await?
} else {
let mut config = ClientConfig::new();
config.alpn_protocols.push(b"h2".to_vec());
config.root_store.add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
config.root_store.add_pem_file(
&mut PubCertificate::get("lightwalletd-lite.myhush.pem").unwrap().as_ref()).unwrap();
if no_cert {
config.dangerous()
.set_certificate_verifier(Arc::new(danger::NoCertificateVerification {}));
}
let tls = ClientTlsConfig::new()
.rustls_client_config(config)
.domain_name(uri.host().unwrap());
Channel::builder(uri.clone())
.tls_config(tls)
.connect()
.await?
};
Ok(CompactTxStreamerClient::new(channel))
}
// ==============
// GRPC code
// ==============
async fn get_lightd_info(uri: &http::Uri, no_cert: bool) -> Result<LightdInfo, Box<dyn std::error::Error>> {
let mut client = get_client(uri, no_cert).await?;
let request = Request::new(Empty {});
let response = client.get_lightd_info(request).await?;
Ok(response.into_inner())
}
pub fn get_info(uri: &http::Uri, no_cert: bool) -> Result<LightdInfo, String> {
let mut rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?;
rt.block_on(get_lightd_info(uri, no_cert)).map_err( |e| e.to_string())
}
async fn get_coinsupply_info(uri: &http::Uri, no_cert: bool) -> Result<Coinsupply, Box<dyn std::error::Error>> {
let mut client = get_client(uri, no_cert).await?;
let request = Request::new(Empty {});
let response = client.get_coinsupply(request).await?;
Ok(response.into_inner())
}
pub fn get_coinsupply(uri: http::Uri, no_cert: bool) -> Result<Coinsupply, String> {
let mut rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?;
rt.block_on(get_coinsupply_info(&uri, no_cert)).map_err( |e| e.to_string())
}
async fn get_block_range<F : 'static + std::marker::Send>(
uri: &http::Uri,
start_height: u64,
end_height: u64,
no_cert: bool,
pool: ThreadPool,
c: F
) -> Result<(), Box<dyn std::error::Error>>
where F : Fn(&[u8], u64) {
let mut client = get_client(uri, no_cert).await?;
let bs = BlockId { height: start_height, hash: vec![] };
let be = BlockId { height: end_height, hash: vec![] };
let request = Request::new(BlockRange { start: Some(bs), end: Some(be) });
let (tx, rx) = channel::<Option<CompactBlock>>();
let (ftx, frx) = channel();
pool.execute(move || {
while let Ok(Some(block)) = rx.recv() {
use prost::Message;
let mut encoded_buf = vec![];
if let Err(e) = block.encode(&mut encoded_buf) {
error!("Error encoding block: {:?}", e);
break;
}
c(&encoded_buf, block.height);
}
if let Err(e) = ftx.send(Ok(())) {
error!("Error sending completion signal: {:?}", e);
}
});
let mut response = client.get_block_range(request).await?.into_inner();
while let Some(block) = response.message().await? {
if let Err(e) = tx.send(Some(block)) {
error!("Error sending block to channel: {:?}", e);
break;
}
}
if let Err(e) = tx.send(None) {
error!("Error sending end signal to channel: {:?}", e);
}
frx.iter().take(1).collect::<Result<Vec<()>, String>>()?;
Ok(())
}
pub fn fetch_blocks<F : 'static + std::marker::Send>(uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) -> Result<(), String>
where F : Fn(&[u8], u64) {
let mut rt = tokio::runtime::Runtime::new().map_err(|e| format!("Error creating runtime {:?}", e))?;
fetch_blocks_with_runtime(&mut rt, uri, start_height, end_height, no_cert, pool, c)
}
pub fn fetch_blocks_with_runtime<F : 'static + std::marker::Send>(rt: &mut tokio::runtime::Runtime, uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) -> Result<(), String>
where F : Fn(&[u8], u64) {
match rt.block_on(get_block_range(uri, start_height, end_height, no_cert, pool, c)) {
Ok(o) => Ok(o),
Err(e) => {
let e = format!("Error fetching blocks {:?}", e);
error!("{}", e);
Err(e)
}
}
}
// get_address_txids GRPC call
async fn get_address_txids<F : 'static + std::marker::Send>(
uri: &http::Uri,
address: String,
start_height: u64,
end_height: u64,
no_cert: bool,
c: F
) -> Result<(), Box<dyn std::error::Error>>
where F : Fn(&[u8], u64) {
let mut client = match get_client(uri, no_cert).await {
Ok(client) => client,
Err(e) => {
error!("Error creating client: {:?}", e);
return Err(e.into());
}
};
let start = Some(BlockId{ height: start_height, hash: vec!()});
let end = Some(BlockId{ height: end_height, hash: vec!()});
let request = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{ start, end }) });
let maybe_response = match client.get_address_txids(request).await {
Ok(response) => response,
Err(e) => {
error!("Error getting address txids: {:?}", e);
return Err(e.into());
}
};
let mut response = maybe_response.into_inner();
while let Some(tx) = response.message().await? {
c(&tx.data, tx.height);
}
Ok(())
}
// function to monitor mempool transactions
pub async fn monitor_mempool<F: 'static + std::marker::Send>(
uri: &http::Uri,
no_cert: bool,
mut c: F
) -> Result<(), Box<dyn std::error::Error>>
where
F: FnMut(RawTransaction) -> Result<(), Box<dyn std::error::Error>>,
{
let mut client = get_client(uri, no_cert)
.await
.map_err(|e| format!("Error getting client: {:?}", e))?;
let request = Request::new(Empty {});
let mut response = client
.get_mempool_stream(request)
.await
.map_err(|e| format!("{}", e))?
.into_inner();
while let Ok(Some(rtx)) = response.message().await {
if let Err(e) = c(rtx) {
info!("Error processing RawTransaction: {:?}", e);
}
}
Ok(())
}
pub fn fetch_transparent_txids<F : 'static + std::marker::Send>(
uri: &http::Uri,
address: String,
start_height: u64,
end_height: u64,
no_cert: bool,
c: F
) -> Result<(), String>
where F : Fn(&[u8], u64) {
let mut rt = match tokio::runtime::Runtime::new() {
Ok(r) => r,
Err(e) => {
let e = format!("Error creating runtime {:?}", e);
error!("{}", e);
return Err(e);
}
};
match rt.block_on(get_address_txids(uri, address.clone(), start_height, end_height, no_cert, c)) {
Ok(o) => Ok(o),
Err(e) => {
let e = format!("Error with get_address_txids runtime {:?}", e);
error!("{}", e);
return Err(e)
}
}
}
// get_transaction GRPC call
async fn get_transaction(uri: &http::Uri, txid: TxId, no_cert: bool)
-> Result<RawTransaction, Box<dyn std::error::Error>> {
let mut client = get_client(uri, no_cert).await?;
let request = Request::new(TxFilter { block: None, index: 0, hash: txid.0.to_vec() });
let response = client.get_transaction(request).await?;
Ok(response.into_inner())
}
pub fn fetch_full_tx(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result<Vec<u8>, String> {
let mut rt = match tokio::runtime::Runtime::new() {
Ok(r) => r,
Err(e) => {
let errstr = format!("Error creating runtime {}", e.to_string());
error!("{}", errstr);
return Err(errstr);
}
};
match rt.block_on(get_transaction(uri, txid, no_cert)) {
Ok(rawtx) => Ok(rawtx.data.to_vec()),
Err(e) => {
let errstr = format!("Error in get_transaction runtime {}", e.to_string());
error!("{}", errstr);
Err(errstr)
}
}
}
/// Shared-connection fetcher for fulltx_fetch::run_fetch_scan. Every concurrent GetTransaction is
/// multiplexed over ONE tonic Channel (cheap Clone; one TLS + HTTP/2 connection) instead of a new
/// Tokio runtime + TLS connection per txid (fetch_full_tx). The channel is dialed lazily on first
/// use, inside the caller's runtime (Channel spawns its buffer task there), and re-dialed when the
/// engine retries with reconnect == true. Error text matches fetch_full_tx.
pub fn make_shared_tx_fetcher(uri: http::Uri, no_cert: bool) -> crate::fulltx_fetch::FetchFn {
let slot: Arc<std::sync::Mutex<Option<CompactTxStreamerClient<Channel>>>> =
Arc::new(std::sync::Mutex::new(None));
Arc::new(move |txid: TxId, reconnect: bool| {
let slot = slot.clone();
let uri = uri.clone();
Box::pin(async move {
let cached = if reconnect {
None
} else {
match slot.lock() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
};
let mut client = match cached {
Some(c) => c,
None => {
let c = match get_client(&uri, no_cert).await {
Ok(c) => c,
Err(e) => return Err(format!("Error in get_transaction runtime {}", e.to_string())),
};
match slot.lock() {
Ok(mut guard) => *guard = Some(c.clone()),
Err(poisoned) => *poisoned.into_inner() = Some(c.clone()),
}
c
}
};
let request = Request::new(TxFilter { block: None, index: 0, hash: txid.0.to_vec() });
match client.get_transaction(request).await {
Ok(response) => Ok(response.into_inner().data.to_vec()),
Err(e) => Err(format!("Error in get_transaction runtime {}", e.to_string())),
}
}) as crate::fulltx_fetch::FetchFuture
})
}
// send_transaction GRPC call
async fn send_transaction(uri: &http::Uri, no_cert: bool, tx_bytes: Box<[u8]>) -> Result<String, Box<dyn std::error::Error>> {
let mut client = get_client(uri, no_cert).await?;
let request = Request::new(RawTransaction {data: tx_bytes.to_vec(), height: 0});
let response = client.send_transaction(request).await?;
let sendresponse = response.into_inner();
if sendresponse.error_code == 0 {
let mut txid = sendresponse.error_message;
if txid.starts_with("\"") && txid.ends_with("\"") {
txid = txid[1..txid.len()-1].to_string();
}
Ok(txid)
} else {
Err(Box::from(format!("Error: {:?}", sendresponse)))
}
}
pub fn broadcast_raw_tx(uri: &http::Uri, no_cert: bool, tx_bytes: Box<[u8]>) -> Result<String, String> {
let mut rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?;
rt.block_on(send_transaction(uri, no_cert, tx_bytes)).map_err( |e| e.to_string())
}
// get_latest_block GRPC call
async fn get_latest_block(uri: &http::Uri, no_cert: bool) -> Result<BlockId, Box<dyn std::error::Error>> {
let mut client = get_client(uri, no_cert).await?;
let request = Request::new(ChainSpec {});
let response = client.get_latest_block(request).await?;
Ok(response.into_inner())
}
pub fn fetch_latest_block(uri: &http::Uri, no_cert: bool) -> Result<BlockId, String> {
let mut rt = match tokio::runtime::Runtime::new() {
Ok(r) => r,
Err(e) => {
let errstr = format!("Error creating runtime {}", e.to_string());
error!("{}", errstr);
return Err(errstr);
}
};
rt.block_on(get_latest_block(uri, no_cert)).map_err(|e| {
let errstr = format!("Error getting latest block {}", e.to_string());
error!("{}", errstr);
errstr
})
}