From 35ce90eec969732839b2a20a564c617b7608a31e Mon Sep 17 00:00:00 2001 From: Deniod Date: Fri, 1 Dec 2023 08:57:19 +0100 Subject: [PATCH 01/19] prevent unwrap(), poisoned data --- lib/src/lightwallet.rs | 45 +++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 3271a95..76d7f17 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -1470,26 +1470,43 @@ pub fn scan_full_tx(&self, tx: &Transaction, height: i32, datetime: u64) { // Do it in a short scope because of the write lock. { info!("A sapling output was sent in {}", tx.txid()); - let mut txs = self.txs.write().unwrap(); - if txs.get(&tx.txid()).unwrap().outgoing_metadata.iter() - .find(|om| om.address == address && om.value == note.value && om.memo == memo) - .is_some() { - warn!("Duplicate outgoing metadata"); - continue; - } + match self.txs.write() { + Ok(mut txs) => { + match txs.get(&tx.txid()) { + Some(wtx) => { + if wtx.outgoing_metadata.iter() + .any(|om| om.address == address && om.value == note.value && om.memo == memo) + { + warn!("Duplicate outgoing metadata"); + continue; + } - // Write the outgoing metadata - txs.get_mut(&tx.txid()).unwrap() - .outgoing_metadata - .push(OutgoingTxMetadata{ - address, value: note.value, memo, - }); + // Write the outgoing metadata + txs.get_mut(&tx.txid()).unwrap() + .outgoing_metadata + .push(OutgoingTxMetadata { + address, + value: note.value, + memo, + }); + }, + None => { + error!("Can not find any entry for txid : {}", tx.txid()); + continue; + } + } + }, + Err(poisoned) => { + error!("Lock is poisoned: {}", poisoned); + return; + } + } } }, None => {} }; } - } + } // Mark this Tx as scanned { let mut txs = self.txs.write().unwrap(); From feb26dbdfb778cd8e2a1d1a51bae45a8a0f5e747 Mon Sep 17 00:00:00 2001 From: Deniod Date: Sat, 13 Jan 2024 21:55:47 +0100 Subject: [PATCH 02/19] prevent unwrap in scan_taddress_txids --- lib/src/lightclient.rs | 93 +++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index 4f53169..09d8882 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -1477,47 +1477,84 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } } } - - fn scan_taddress_txids(&self, pool: &ThreadPool, block_times: Arc>>, start_height: u64, end_height: u64, no_cert: bool) -> Result, String> { - // Copy over addresses so as to not lock up the wallet, which we'll use inside the callback below. - let addresses = self.wallet.read().unwrap() + fn scan_taddress_txids( + &self, + pool: &ThreadPool, + block_times: Arc>>, + start_height: u64, + end_height: u64, + no_cert: bool + ) -> Result, String> { + let addresses = self.wallet.read() + .map_err(|e| format!("Failed to read wallet: {:?}", e))? .get_all_taddresses().iter() - .map(|a| a.clone()) + .cloned() .collect::>(); - - // Create a channel so the fetch_transparent_txids can send the results back + let (ctx, crx) = channel(); let num_addresses = addresses.len(); - + for address in addresses { + let address_clone = address.clone(); let wallet = self.wallet.clone(); - let pool = pool.clone(); let server_uri = self.get_server_uri(); let ctx = ctx.clone(); - let block_times = block_times.clone(); - + pool.execute(move || { - // Fetch the transparent transactions for this address, and send the results - // via the channel - let r = fetch_transparent_txids(&server_uri, address, start_height, end_height,no_cert, - move |tx_bytes: &[u8], height: u64| { - let tx = Transaction::read(tx_bytes).unwrap(); - - // Scan this Tx for transparent inputs and outputs - let datetime = block_times.read().unwrap().get(&height).map(|v| *v).unwrap_or(0); - wallet.read().unwrap().scan_full_tx(&tx, height as i32, datetime as u64); - }); - ctx.send(r).unwrap(); + println!("Fetching transactions for address: {}", address_clone); + + let r = fetch_transparent_txids( + &server_uri, + address, + start_height, + end_height, + no_cert, + move |tx_bytes: &[u8], height: u64| { + let tx_result = Transaction::read(tx_bytes) + .map_err(|e| format!("Failed to read transaction: {:?}", e)); + + match tx_result { + Ok(tx) => { + let datetime_result = block_times.read() + .map_err(|e| format!("Failed to read block times: {:?}", e)) + .and_then(|bt| bt.get(&height).cloned().ok_or_else(|| format!("No datetime for height: {}", height))); + + match datetime_result { + Ok(datetime) => { + match wallet.read().map_err(|e| format!("Failed to read wallet: {:?}", e)) { + Ok(w) => { + w.scan_full_tx(&tx, height as i32, datetime as u64); + }, + Err(e) => { + println!("Error reading wallet: {}", e); + }, + } + }, + Err(e) => { + println!("Error processing transaction: {}", e); + }, + } + }, + Err(e) => { + println!("Error reading transaction: {}", e); + }, + } + } + ); + + match ctx.send(r) { + Ok(_) => println!("Successfully sent data for address: {}", address_clone), + Err(e) => println!("Failed to send data for address: {}: {:?}", address_clone, e), + } }); } - - // Collect all results from the transparent fetches, and make sure everything was OK. - // If it was not, we return an error, which will go back to the retry - crx.iter().take(num_addresses).collect::, String>>() - } - + + crx.iter().take(num_addresses).collect() + } + + fn scan_fill_fulltxs(&self, pool: &ThreadPool, decoy_txids: Vec<(TxId, i32)>) -> Result, String> { // We need to first copy over the Txids from the wallet struct, because From c8948af2fcb161d6a11017432f9ced39052e3d60 Mon Sep 17 00:00:00 2001 From: Deniod Date: Sat, 13 Jan 2024 22:12:32 +0100 Subject: [PATCH 03/19] prevent unwrap in get_block_range --- lib/src/grpcconnector.rs | 72 ++++++++++++++++++++++++---------------- lib/src/lightclient.rs | 5 ++- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/lib/src/grpcconnector.rs b/lib/src/grpcconnector.rs index 4419c0a..061b4d2 100644 --- a/lib/src/grpcconnector.rs +++ b/lib/src/grpcconnector.rs @@ -100,49 +100,63 @@ pub fn get_coinsupply(uri: http::Uri, no_cert: bool) -> Result(uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) - -> Result<(), Box> +async fn get_block_range( + uri: &http::Uri, + start_height: u64, + end_height: u64, + no_cert: bool, + pool: ThreadPool, + c: F +) -> Result<(), Box> where F : Fn(&[u8], u64) { let mut client = get_client(uri, no_cert).await?; - let bs = BlockId{ height: start_height, hash: vec!()}; - let be = BlockId{ height: end_height, hash: vec!()}; + let bs = BlockId { height: start_height, hash: vec![] }; + let be = BlockId { height: end_height, hash: vec![] }; - let request = Request::new(BlockRange{ start: Some(bs), end: Some(be) }); + let request = Request::new(BlockRange { start: Some(bs), end: Some(be) }); - // Channel where the blocks are sent. A None signifies end of all blocks - let (tx, rx) = channel::>(); + let (tx, rx) = channel::>(); + let (ftx, frx) = channel(); - // Channel that the processor signals it is done, so the method can return - let (ftx, frx) = channel(); - - // The processor runs on a different thread, so that the network calls don't - // block on this - pool.execute(move || { - while let Some(block) = rx.recv().unwrap() { - use prost::Message; - let mut encoded_buf = vec![]; - - block.encode(&mut encoded_buf).unwrap(); - c(&encoded_buf, block.height); - } - - ftx.send(Ok(())).unwrap(); - }); + pool.execute(move || { + while let Ok(Some(block)) = rx.recv() { + use prost::Message; + let mut encoded_buf = vec![]; + + match block.encode(&mut encoded_buf) { + Ok(_) => c(&encoded_buf, block.height), + Err(e) => { + eprintln!("Error encoding block: {:?}", e); + break; + } + } + } + + if let Err(e) = ftx.send(Ok(())) { + eprintln!("Error sending completion signal: {:?}", e); + } + }); let mut response = client.get_block_range(request).await?.into_inner(); - //println!("{:?}", response); - while let Some(block) = response.message().await? { - tx.send(Some(block)).unwrap(); - } - tx.send(None).unwrap(); - // Wait for the processor to exit + while let Some(block) = response.message().await? { + if let Err(e) = tx.send(Some(block)) { + eprintln!("Error sending block to channel: {:?}", e); + break; + } + } + + if let Err(e) = tx.send(None) { + eprintln!("Error sending end signal to channel: {:?}", e); + } + frx.iter().take(1).collect::, String>>()?; Ok(()) } + pub fn fetch_blocks(uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) -> Result<(), String> where F : Fn(&[u8], u64) { diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index 09d8882..c3c6de0 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -1503,8 +1503,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { let block_times = block_times.clone(); pool.execute(move || { - println!("Fetching transactions for address: {}", address_clone); - + let r = fetch_transparent_txids( &server_uri, address, @@ -1545,7 +1544,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { ); match ctx.send(r) { - Ok(_) => println!("Successfully sent data for address: {}", address_clone), + Ok(_) => info!("Successfully sent data for address: {}", address_clone), Err(e) => println!("Failed to send data for address: {}: {:?}", address_clone, e), } }); From 60550982e0dd686b8ec940524091eb67667eff14 Mon Sep 17 00:00:00 2001 From: Deniod Date: Sat, 13 Jan 2024 22:15:31 +0100 Subject: [PATCH 04/19] less debug --- lib/src/grpcconnector.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/src/grpcconnector.rs b/lib/src/grpcconnector.rs index 061b4d2..63db47a 100644 --- a/lib/src/grpcconnector.rs +++ b/lib/src/grpcconnector.rs @@ -97,7 +97,6 @@ pub fn get_coinsupply(uri: http::Uri, no_cert: bool) -> Result( @@ -165,7 +164,6 @@ pub fn fetch_blocks(uri: &http::Uri, start_heig Err(e) => { let es = format!("Error creating runtime {:?}", e); error!("{}", es); - eprintln!("{}", e); return Err(es); } }; @@ -175,7 +173,6 @@ pub fn fetch_blocks(uri: &http::Uri, start_heig Err(e) => { let e = format!("Error fetching blocks {:?}", e); error!("{}", e); - eprintln!("{}", e); Err(e) } } @@ -246,7 +243,6 @@ pub fn fetch_transparent_txids(uri: &http::Uri, Err(e) => { let e = format!("Error creating runtime {:?}", e); error!("{}", e); - eprintln!("{}", e); return Err(e); } }; @@ -256,13 +252,11 @@ pub fn fetch_transparent_txids(uri: &http::Uri, Err(e) => { let e = format!("Error with get_address_txids runtime {:?}", e); error!("{}", e); - eprintln!("{}", e); Err(e) } } } - // get_transaction GRPC call async fn get_transaction(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result> { @@ -280,7 +274,6 @@ pub fn fetch_full_tx(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result { let errstr = format!("Error creating runtime {}", e.to_string()); error!("{}", errstr); - eprintln!("{}", errstr); return Err(errstr); } }; @@ -290,7 +283,6 @@ pub fn fetch_full_tx(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result { let errstr = format!("Error in get_transaction runtime {}", e.to_string()); error!("{}", errstr); - eprintln!("{}", errstr); Err(errstr) } } From a307f828c98ae7515e65b2d2ea7d58ffcac8f693 Mon Sep 17 00:00:00 2001 From: Deniod Date: Sat, 13 Jan 2024 22:41:02 +0100 Subject: [PATCH 05/19] less debug, handle error if channel is closed --- lib/src/grpcconnector.rs | 70 +++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/lib/src/grpcconnector.rs b/lib/src/grpcconnector.rs index 63db47a..f7d9894 100644 --- a/lib/src/grpcconnector.rs +++ b/lib/src/grpcconnector.rs @@ -122,21 +122,20 @@ where F : Fn(&[u8], u64) { while let Ok(Some(block)) = rx.recv() { use prost::Message; let mut encoded_buf = vec![]; - - match block.encode(&mut encoded_buf) { - Ok(_) => c(&encoded_buf, block.height), - Err(e) => { - eprintln!("Error encoding block: {:?}", e); - break; - } + + if let Err(e) = block.encode(&mut encoded_buf) { + eprintln!("Error encoding block: {:?}", e); + break; } + + c(&encoded_buf, block.height); } - + if let Err(e) = ftx.send(Ok(())) { eprintln!("Error sending completion signal: {:?}", e); } }); - + let mut response = client.get_block_range(request).await?.into_inner(); while let Some(block) = response.message().await? { @@ -178,25 +177,44 @@ pub fn fetch_blocks(uri: &http::Uri, start_heig } } - // get_address_txids GRPC call -async fn get_address_txids(uri: &http::Uri, address: String, - start_height: u64, end_height: u64, no_cert: bool, c: F) -> Result<(), Box> - where F : Fn(&[u8], u64) { +async fn get_address_txids( + uri: &http::Uri, + address: String, + start_height: u64, + end_height: u64, + no_cert: bool, + c: F +) -> Result<(), Box> +where F : Fn(&[u8], u64) { + + let mut client = match get_client(uri, no_cert).await { + Ok(client) => client, + Err(e) => { + eprintln!("Error creating client: {:?}", e); + return Err(e.into()); + } + }; - let mut client = get_client(uri, no_cert).await?; let start = Some(BlockId{ height: start_height, hash: vec!()}); - let end = Some(BlockId{ height: end_height, hash: vec!()}); + let end = Some(BlockId{ height: end_height, hash: vec!()}); - let request = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{start, end}) }); + let request = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{ start, end }) }); + + let maybe_response = match client.get_address_txids(request).await { + Ok(response) => response, + Err(e) => { + eprintln!("Error getting address txids: {:?}", e); + return Err(e.into()); + } + }; - let maybe_response = client.get_address_txids(request).await?; let mut response = maybe_response.into_inner(); while let Some(tx) = response.message().await? { c(&tx.data, tx.height); } - + Ok(()) } @@ -234,10 +252,16 @@ where Ok(()) } -pub fn fetch_transparent_txids(uri: &http::Uri, address: String, - start_height: u64, end_height: u64, no_cert: bool, c: F) -> Result<(), String> - where F : Fn(&[u8], u64) { - +pub fn fetch_transparent_txids( + uri: &http::Uri, + address: String, + start_height: u64, + end_height: u64, + no_cert: bool, + c: F +) -> Result<(), String> +where F : Fn(&[u8], u64) { + let mut rt = match tokio::runtime::Runtime::new() { Ok(r) => r, Err(e) => { @@ -252,7 +276,7 @@ pub fn fetch_transparent_txids(uri: &http::Uri, Err(e) => { let e = format!("Error with get_address_txids runtime {:?}", e); error!("{}", e); - Err(e) + return Err(e) } } } From 1f184f55d474b93464fdba0ee29bf54817e9d389 Mon Sep 17 00:00:00 2001 From: Deniod Date: Sat, 13 Jan 2024 22:43:49 +0100 Subject: [PATCH 06/19] less debug --- lib/src/lightclient.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index c3c6de0..e698c95 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -1233,7 +1233,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } // Sleep exponentially backing off std::thread::sleep(std::time::Duration::from_secs((2 as u64).pow(retry_count))); - println!("Sync error {}\nRetry count {}", e, retry_count); + // println!("Sync error {}\nRetry count {}", e, retry_count); } } } From 568d7b7860ec0b848cb84474f38b25bed890de09 Mon Sep 17 00:00:00 2001 From: lucretius Date: Mon, 15 Jan 2024 17:46:10 +0100 Subject: [PATCH 07/19] update proto files --- lib/proto/compact_formats.proto | 45 +++++++++----- lib/proto/service.proto | 100 +++++++++++++++++++++++++++----- 2 files changed, 115 insertions(+), 30 deletions(-) diff --git a/lib/proto/compact_formats.proto b/lib/proto/compact_formats.proto index 7e1bc54..cecce25 100644 --- a/lib/proto/compact_formats.proto +++ b/lib/proto/compact_formats.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package cash.z.wallet.sdk.rpc; -option go_package = "walletrpc"; +option go_package = "lightwalletd/walletrpc"; +option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. @@ -11,11 +12,11 @@ option go_package = "walletrpc"; // 3. Update your witnesses to generate new Sapling spend proofs. message CompactBlock { uint32 protoVersion = 1; // the version of this wire format, for storage - uint64 height = 2; // the height of this block - bytes hash = 3; - bytes prevHash = 4; - uint32 time = 5; - bytes header = 6; // (hash, prevHash, and time) OR (full header) + uint64 height = 2; // the height of this block + bytes hash = 3; // the ID (hash) of this block, same as in block explorers + bytes prevHash = 4; // the ID (hash) of this block's predecessor + uint32 time = 5; // Unix epoch time when the block was mined + bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // compact transactions from this block } @@ -23,8 +24,8 @@ message CompactTx { // Index and hash will allow the receiver to call out to chain // explorers or other data structures to retrieve more information // about this transaction. - uint64 index = 1; - bytes hash = 2; + uint64 index = 1; // the index within the full block + bytes hash = 2; // the ID (hash) of this transaction, same as in block explorers // The transaction fee: present if server can provide. In the case of a // stateless server and a transaction with transparent inputs, this will be @@ -33,16 +34,32 @@ message CompactTx { // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) uint32 fee = 3; - repeated CompactSpend spends = 4; - repeated CompactOutput outputs = 5; + repeated CompactSaplingSpend spends = 4; + repeated CompactSaplingOutput outputs = 5; } +// CompactSaplingSpend is a Sapling Spend Description as described in 7.3 of the Zcash +// protocol specification. +message CompactSaplingSpend { + bytes nf = 1; // nullifier (see the Zcash protocol specification) +} + +// output is a Sapling Output Description as described in section 7.4 of the +// Zcash protocol spec. Total size is 948. +message CompactSaplingOutput { + bytes cmu = 1; // note commitment u-coordinate + bytes epk = 2; // ephemeral public key + bytes ciphertext = 3; // first 52 bytes of ciphertext +} + +/* message CompactSpend { - bytes nf = 1; + bytes nf = 1; // nullifier (see the Zcash protocol specification) } message CompactOutput { - bytes cmu = 1; - bytes epk = 2; - bytes ciphertext = 3; + bytes cmu = 1; // note commitment u-coordinate + bytes epk = 2; // ephemeral public key + bytes ciphertext = 3; // first 52 bytes of ciphertext } +*/ diff --git a/lib/proto/service.proto b/lib/proto/service.proto index e5719ad..aa52e39 100644 --- a/lib/proto/service.proto +++ b/lib/proto/service.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package cash.z.wallet.sdk.rpc; -option go_package = "walletrpc"; - +option go_package = "lightwalletd/walletrpc"; +option swift_prefix = ""; import "compact_formats.proto"; // A BlockID message contains identifiers to select a block: a height or a @@ -56,14 +56,13 @@ message LightdInfo { uint64 longestchain = 9; uint64 notarized = 10; } - message Coinsupply { string result = 1; string coin = 2; - int64 height = 3; - int64 supply = 4; - int64 zfunds = 5; - int64 total = 6; + uint64 height = 3; + uint64 supply = 4; + uint64 zfunds = 5; + uint64 total = 6; } message TransparentAddress { @@ -75,28 +74,97 @@ message TransparentAddressBlockFilter { BlockRange range = 2; } +message Address { + string address = 1; +} +message AddressList { + repeated string addresses = 1; +} +message Balance { + int64 valueZat = 1; +} + message Exclude { repeated bytes txid = 1; } +// The TreeState is derived from the Hush getblockmerkletree rpc. +// https://faq.hush.is/rpc/getblockmerkletree.html +message TreeState { + string network = 1; // "main" or "test" + uint64 height = 2; // block height + string hash = 3; // block id + uint32 time = 4; // Unix epoch time when the block was mined + string saplingTree = 5; // sapling commitment tree state +} + +// Results are sorted by height, which makes it easy to issue another +// request that picks up from where the previous left off. +message GetAddressUtxosArg { + repeated string addresses = 1; + uint64 startHeight = 2; + uint32 maxEntries = 3; // zero means unlimited +} +message GetAddressUtxosReply { + string address = 6; + bytes txid = 1; + int32 index = 2; + bytes script = 3; + int64 valueZat = 4; + uint64 height = 5; +} +message GetAddressUtxosReplyList { + repeated GetAddressUtxosReply addressUtxos = 1; +} + service CompactTxStreamer { - // Compact Blocks + // Return the height of the tip of the best chain rpc GetLatestBlock(ChainSpec) returns (BlockID) {} + // Return the compact block corresponding to the given block identifier rpc GetBlock(BlockID) returns (CompactBlock) {} + // Return a list of consecutive compact blocks rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {} - // Transactions + // Return the requested full (not compact) transaction (as from zcashd) rpc GetTransaction(TxFilter) returns (RawTransaction) {} + // Submit the given transaction to the Zcash network rpc SendTransaction(RawTransaction) returns (SendResponse) {} - // t-Address support + // Return the txids corresponding to the given t-address within the given block range + rpc GetTaddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} + // wrapper for GetTaddressTxids rpc GetAddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} + rpc GetTaddressBalance(AddressList) returns (Balance) {} + rpc GetTaddressBalanceStream(stream Address) returns (Balance) {} - // Misc - rpc GetLightdInfo(Empty) returns (LightdInfo) {} - rpc GetCoinsupply(Empty) returns (Coinsupply) {} - - //Mempool + // Return the compact transactions currently in the mempool; the results + // can be a few seconds out of date. If the Exclude list is empty, return + // all transactions; otherwise return all *except* those in the Exclude list + // (if any); this allows the client to avoid receiving transactions that it + // already has (from an earlier call to this rpc). The transaction IDs in the + // Exclude list can be shortened to any number of bytes to make the request + // more bandwidth-efficient; if two or more transactions in the mempool + // match a shortened txid, they are all sent (none is excluded). Transactions + // in the exclude list that don't exist in the mempool are ignored. rpc GetMempoolTx(Exclude) returns (stream CompactTx) {} + + // Return a stream of current Mempool transactions. This will keep the output stream open while + // there are mempool transactions. It will close the returned stream when a new block is mined. rpc GetMempoolStream(Empty) returns (stream RawTransaction) {} -} + + // GetTreeState returns the note commitment tree state corresponding to the given block. + // See section 3.7 of the Zcash protocol specification. It returns several other useful + // values also (even though they can be obtained using GetBlock). + // The block can be specified by either height or hash. + rpc GetTreeState(BlockID) returns (TreeState) {} + rpc GetLatestTreeState(Empty) returns (TreeState) {} + + rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {} + rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {} + + // Return information about this lightwalletd instance and the blockchain + rpc GetLightdInfo(Empty) returns (LightdInfo) {} + // Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) + // rpc Ping(Duration) returns (PingResponse) {} + rpc GetCoinsupply(Empty) returns (Coinsupply) {} +} \ No newline at end of file From 8076ee1426aa5f0f95100a85404ad6f507a764fd Mon Sep 17 00:00:00 2001 From: lucretius Date: Sat, 20 Jan 2024 15:43:43 +0100 Subject: [PATCH 08/19] add check for wallet version --- lib/src/lightwallet/data.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index eb57b13..696c5f4 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -484,11 +484,11 @@ impl WalletTx { // Outgoing metadata was only added in version 2 let outgoing_metadata = Vector::read(&mut reader, |r| OutgoingTxMetadata::read(r))?; - let incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; + //let incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; let full_tx_scanned = reader.read_u8()? > 0; - Ok(WalletTx{ + let mut wallet_tx = WalletTx { block, datetime, txid, @@ -497,9 +497,15 @@ impl WalletTx { total_shielded_value_spent, total_transparent_value_spent, outgoing_metadata, - incoming_metadata, + incoming_metadata: vec![], full_tx_scanned - }) + }; + if version >= 5 { + wallet_tx.incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; + } + + Ok(wallet_tx) + } pub fn write(&self, mut writer: W) -> io::Result<()> { From e4658582cbe610cb6f7f5d62d2ea9444bd38f4ca Mon Sep 17 00:00:00 2001 From: lucretius Date: Sat, 20 Jan 2024 15:59:30 +0100 Subject: [PATCH 09/19] check for version 5 --- lib/src/lightwallet/data.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index 696c5f4..b8abcd0 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -440,7 +440,7 @@ pub struct WalletTx { impl WalletTx { pub fn serialized_version() -> u64 { - return 4; + return 5; } pub fn new(height: i32, datetime: u64, txid: &TxId) -> Self { @@ -460,10 +460,10 @@ impl WalletTx { pub fn read(mut reader: R) -> io::Result { let version = reader.read_u64::()?; + println!("wallet Version : {}", version); assert!(version <= WalletTx::serialized_version()); let block = reader.read_i32::()?; - let datetime = if version >= 4 { reader.read_u64::()? } else { @@ -472,23 +472,24 @@ impl WalletTx { let mut txid_bytes = [0u8; 32]; reader.read_exact(&mut txid_bytes)?; - let txid = TxId{0: txid_bytes}; let notes = Vector::read(&mut reader, |r| SaplingNoteData::read(r))?; let utxos = Vector::read(&mut reader, |r| Utxo::read(r))?; - let total_shielded_value_spent = reader.read_u64::()?; let total_transparent_value_spent = reader.read_u64::()?; - - // Outgoing metadata was only added in version 2 let outgoing_metadata = Vector::read(&mut reader, |r| OutgoingTxMetadata::read(r))?; - //let incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; + // Read incoming_metadata only if version is 5 or higher + let incoming_metadata = if version >= 5 { + Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))? + } else { + vec![] + }; let full_tx_scanned = reader.read_u8()? > 0; - - let mut wallet_tx = WalletTx { + + Ok(WalletTx { block, datetime, txid, @@ -497,17 +498,12 @@ impl WalletTx { total_shielded_value_spent, total_transparent_value_spent, outgoing_metadata, - incoming_metadata: vec![], + incoming_metadata, full_tx_scanned - }; - if version >= 5 { - wallet_tx.incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; - } - - Ok(wallet_tx) - + }) } + pub fn write(&self, mut writer: W) -> io::Result<()> { writer.write_u64::(WalletTx::serialized_version())?; From b3e4162f8a1b5161d02e56199dab95fc1cf93e20 Mon Sep 17 00:00:00 2001 From: lucretius Date: Sat, 20 Jan 2024 16:05:35 +0100 Subject: [PATCH 10/19] add error warning --- lib/src/lightwallet/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index b8abcd0..dcc0007 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -461,7 +461,7 @@ impl WalletTx { pub fn read(mut reader: R) -> io::Result { let version = reader.read_u64::()?; println!("wallet Version : {}", version); - assert!(version <= WalletTx::serialized_version()); + assert!(version <= WalletTx::serialized_version(), "Version mismatch. Please restore your Seed"); let block = reader.read_i32::()?; let datetime = if version >= 4 { From d8994eec8e285c70f0f26045a377eccc8265c2c4 Mon Sep 17 00:00:00 2001 From: lucretius Date: Sat, 20 Jan 2024 19:47:40 +0100 Subject: [PATCH 11/19] less debug --- lib/src/lightwallet/data.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index dcc0007..657f342 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -460,8 +460,7 @@ impl WalletTx { pub fn read(mut reader: R) -> io::Result { let version = reader.read_u64::()?; - println!("wallet Version : {}", version); - assert!(version <= WalletTx::serialized_version(), "Version mismatch. Please restore your Seed"); + assert!(version <= WalletTx::serialized_version(), "Version mismatch. Please restore with your Seed"); let block = reader.read_i32::()?; let datetime = if version >= 4 { From 3634a6bcdd398fbf48c0783639133a035435f0de Mon Sep 17 00:00:00 2001 From: lucretius Date: Tue, 23 Jan 2024 16:59:54 +0100 Subject: [PATCH 12/19] add custom fee --- Cargo.lock | 18 ++++--- lib/Cargo.toml | 12 ++--- lib/src/commands.rs | 107 +++++++++++++++++++++++------------------ lib/src/lightclient.rs | 5 +- lib/src/lightwallet.rs | 52 ++++++++++++-------- 5 files changed, 113 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1317c15..fab19ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler32" version = "1.0.4" @@ -186,7 +188,7 @@ checksum = "9e0089c35ab7c6f2bc55ab23f769913f0ac65b1023e7e74638a1f43128dd5df2" [[package]] name = "bellman" version = "0.1.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bit-vec", "blake2s_simd", @@ -510,7 +512,7 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "ff" version = "0.4.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "byteorder", "ff_derive", @@ -520,7 +522,7 @@ dependencies = [ [[package]] name = "ff_derive" version = "0.3.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "num-bigint", "num-integer", @@ -678,7 +680,7 @@ dependencies = [ [[package]] name = "group" version = "0.1.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "ff", "rand 0.7.2", @@ -1158,7 +1160,7 @@ dependencies = [ [[package]] name = "pairing" version = "0.14.2" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "byteorder", "ff", @@ -2830,7 +2832,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bech32", "bs58", @@ -2846,7 +2848,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "aes", "blake2b_simd", @@ -2869,7 +2871,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bellman", "blake2b_simd", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 95388cc..924d436 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -43,34 +43,34 @@ webpki-roots = "0.18.0" [dependencies.bellman] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false features = ["groth16"] [dependencies.pairing] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" [dependencies.zcash_client_backend] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false [dependencies.zcash_primitives] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false features = ["transparent-inputs"] [dependencies.zcash_proofs] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false [dependencies.ff] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" features = ["ff_derive"] [build-dependencies] diff --git a/lib/src/commands.rs b/lib/src/commands.rs index b4835ad..4f2b12a 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -5,6 +5,8 @@ use json::{object}; use crate::lightclient::LightClient; use crate::lightwallet::LightWallet; +use zcash_primitives::transaction::components::amount::DEFAULT_FEE; +use std::convert::TryInto; pub trait Command { fn help(&self) -> String; @@ -489,69 +491,82 @@ impl Command for SendCommand { // Parse the args. There are two argument types. // 1 - A set of 2(+1 optional) arguments for a single address send representing address, value, memo? // 2 - A single argument in the form of a JSON string that is "[{address: address, value: value, memo: memo},...]" - // 1 - Destination address. T or Z address if args.len() < 1 || args.len() > 3 { return self.help(); } - // Check for a single argument that can be parsed as JSON let send_args = if args.len() == 1 { let arg_list = args[0]; - - let json_args = match json::parse(&arg_list) { - Ok(j) => j, - Err(e) => { - let es = format!("Couldn't understand JSON: {}", e); - return format!("{}\n{}", es, self.help()); - } - }; - - if !json_args.is_array() { - return format!("Couldn't parse argument as array\n{}", self.help()); + match json::parse(&arg_list) { + Ok(json_args) => { + // Check if the parsed JSON is an array. + if !json_args.is_array() { + return format!("Couldn't parse argument as array\n{}", self.help()); + } + + // Map each JSON object to a tuple (address, amount, memo, fee). + let maybe_send_args = json_args.members().map(|j| { + if !j.has_key("address") || !j.has_key("amount") { + Err(format!("Need 'address' and 'amount'\n")) + } else { + let fee = j["fee"].as_u64().unwrap_or(DEFAULT_FEE.try_into().unwrap()); + Ok(( + j["address"].as_str().unwrap().to_string(), + j["amount"].as_u64().unwrap(), + j["memo"].as_str().map(|s| s.to_string()), + fee + )) + } + }).collect::, u64)>, String>>(); + + // Handle any errors that occurred during mapping. + match maybe_send_args { + Ok(a) => a, + Err(s) => return format!("Error: {}\n{}", s, self.help()), + } + }, + Err(e) => return format!("Couldn't understand JSON: {}\n{}", e, self.help()), } - - let maybe_send_args = json_args.members().map( |j| { - if !j.has_key("address") || !j.has_key("amount") { - Err(format!("Need 'address' and 'amount'\n")) - } else { - Ok((j["address"].as_str().unwrap().to_string().clone(), j["amount"].as_u64().unwrap(), j["memo"].as_str().map(|s| s.to_string().clone()))) - } - }).collect::)>, String>>(); - - match maybe_send_args { - Ok(a) => a.clone(), - Err(s) => { return format!("Error: {}\n{}", s, self.help()); } - } - } else if args.len() == 2 || args.len() == 3 { + } else { + // Handle the case where individual arguments are provided. let address = args[0].to_string(); - - // Make sure we can parse the amount let value = match args[1].parse::() { Ok(amt) => amt, - Err(e) => { - return format!("Couldn't parse amount: {}", e); - } + Err(e) => return format!("Couldn't parse amount: {}", e), }; - - let memo = if args.len() == 3 { Some(args[2].to_string()) } else { None }; - - // Memo has to be None if not sending to a shileded address + let memo = args.get(2).map(|m| m.to_string()); + + // Memo should be None if the address is not shielded. if memo.is_some() && !LightWallet::is_shielded_address(&address, &lightclient.config) { return format!("Can't send a memo to the non-shielded address {}", address); } - - vec![(args[0].to_string(), value, memo)] - } else { - return self.help() + + // Create a vector with a single transaction (address, amount, memo). + vec![(address, value, memo, DEFAULT_FEE.try_into().unwrap())] }; + + // Transform transaction data into the required format (String -> &str). + let tos = send_args.iter().map(|(a, v, m, _)| (a.as_str(), *v, m.clone())).collect::>(); + + // Calculate the total fee for all transactions. + // This assumes that all transactions have the same fee. + // If they can have different fees, you need to modify this logic. + + let default_fee: u64 = DEFAULT_FEE.try_into().unwrap(); + let mut total_fee = default_fee; - // Convert to the right format. String -> &str. - let tos = send_args.iter().map(|(a, v, m)| (a.as_str(), *v, m.clone()) ).collect::>(); - match lightclient.do_send(tos) { - Ok(txid) => { object!{ "txid" => txid } }, - Err(e) => { object!{ "error" => e } } - }.pretty(2) + for (_, _, _, fee) in send_args.iter() { + if *fee != default_fee{ + total_fee = *fee; + break; + } +} + // Execute the transaction and handle the result. + match lightclient.do_send(tos, &total_fee) { + Ok(txid) => object!{ "txid" => txid }.pretty(2), + Err(e) => object!{ "error" => e }.pretty(2), + } } } diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index e698c95..b7c9652 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -1252,6 +1252,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } let addr = address.or(self.wallet.read().unwrap().get_all_zaddresses().get(0).map(|s| s.clone())).unwrap(); + let fee: u64 = DEFAULT_FEE.try_into().unwrap(); let result = { let _lock = self.sync_lock.lock().unwrap(); @@ -1260,6 +1261,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { &self.sapling_spend, &self.sapling_output, true, vec![(&addr, tbal - fee, None)], + &fee, |txbytes| broadcast_raw_tx(&self.get_server_uri(),self.config.no_cert_verification, txbytes) ) }; @@ -1600,7 +1602,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { crx.iter().take(num_fetches).collect::, String>>() } - pub fn do_send(&self, addrs: Vec<(&str, u64, Option)>) -> Result { + pub fn do_send(&self, addrs: Vec<(&str, u64, Option)>, fee: &u64) -> Result { if !self.wallet.read().unwrap().is_unlocked_for_spending() { error!("Wallet is locked"); return Err("Wallet is locked".to_string()); @@ -1616,6 +1618,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { &self.sapling_spend, &self.sapling_output, false, addrs, + fee, |txbytes| broadcast_raw_tx(&self.get_server_uri(), self.config.no_cert_verification, txbytes) ) }; diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 76d7f17..1a2b888 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -39,7 +39,7 @@ use zcash_primitives::{ serialize::{Vector}, transaction::{ builder::{Builder}, - components::{Amount, OutPoint, TxOut}, components::amount::DEFAULT_FEE, + components::{Amount, OutPoint, TxOut}, TxId, Transaction, }, sapling::Node, @@ -2194,6 +2194,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 output_params: &[u8], _transparent_only: bool, tos: Vec<(&str, u64, Option)>, + fee: &u64, broadcast_fn: F ) -> Result<(String, Vec), String> where F: Fn(Box<[u8]>) -> Result @@ -2224,24 +2225,31 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 total_value, tos.len() ); - // Convert address (str) to RecepientAddress and value to Amount - let recepients = tos.iter().map(|to| { - let ra = match address::RecipientAddress::from_str(to.0, - self.config.hrp_sapling_address(), - self.config.base58_pubkey_address(), - self.config.base58_script_address()) { - Some(to) => to, - None => { - let e = format!("Invalid recipient address: '{}'", to.0); - error!("{}", e); - return Err(e); - } - }; + // Convert address (str) to RecipientAddress and value to Amount - let value = Amount::from_u64(to.1).unwrap(); + let recepients: Result)>, String> = tos.iter().map(|to| { + // Convert string to RecipientAddress + let ra = match address::RecipientAddress::from_str( + to.0, + self.config.hrp_sapling_address(), + self.config.base58_pubkey_address(), + self.config.base58_script_address() + ) { + Some(addr) => addr, + None => { + let e = format!("Invalid recipient address: '{}'", to.0); + error!("{}", e); + return Err(e); + } + }; - Ok((ra, value, to.2.clone())) - }).collect::)>, String>>()?; + // Convert the second tuple element to Amount + let value = Amount::from_u64(to.1).expect("Invalid amount value"); + + Ok((ra, value, to.2.clone())) + }).collect(); + + let recepients = recepients?; // Target the next block, assuming we are up-to-date. let (height, anchor_offset) = match self.get_target_height_and_anchor_offset() { @@ -2254,8 +2262,9 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 }; // Select notes to cover the target value - println!("{}: Selecting notes", now() - start_time); - let target_value = Amount::from_u64(total_value).unwrap() + DEFAULT_FEE ; + // Select notes to cover the target value + println!("{}: Selecting notes", now() - start_time); + let target_value = Amount::from_u64(total_value).unwrap() + Amount::from_u64(*fee).unwrap(); // Select the candidate notes that are eligible to be spent let notes: Vec<_> = self.txs.read().unwrap().iter() .map(|(txid, tx)| tx.notes.iter().map(move |note| (*txid, note))) @@ -2340,8 +2349,11 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 return Err(e); } + let fee_amount = Amount::from_u64(*fee).expect("Invalid fee amount"); + builder.set_fee(fee_amount); + // Create the transaction - println!("{}: Adding {} notes and {} utxos", now() - start_time, notes.len(), tinputs.len()); + println!("{}: Adding {} notes and {} utxos and fee {:?}", now() - start_time, notes.len(), tinputs.len(), fee_amount); for selected in notes.iter() { if let Err(e) = builder.add_sapling_spend( From 8be65b967a46e699c07c8a378d40ee4436fe8388 Mon Sep 17 00:00:00 2001 From: lucretius Date: Tue, 23 Jan 2024 17:04:35 +0100 Subject: [PATCH 13/19] get the position for incoming mempool txids --- lib/src/lightclient.rs | 4 ++-- lib/src/lightwallet.rs | 18 +++++++++--------- lib/src/lightwallet/data.rs | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index b7c9652..06f004e 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -953,11 +953,11 @@ impl LightClient { wtxs.iter().flat_map(|wtx| { wtx.incoming_metadata.iter() .enumerate() - .map(move |(_i, om)| + .map(move |(i, om)| object! { "block_height" => wtx.block.clone(), "datetime" => wtx.datetime.clone(), - "position" => om.position, + "position" => i, "txid" => format!("{}", wtx.txid), "amount" => om.value as i64, "address" => om.address.clone(), diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 1a2b888..80a2c0c 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -1562,20 +1562,20 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 return; } - let position = if formatted_memo.as_ref().map_or(false, |m| m.starts_with('{')) { + /*let position = if formatted_memo.as_ref().map_or(false, |m| m.starts_with('{')) { 1 } else { existing_txs.iter() .filter(|tx| !LightWallet::memo_str(&Some(tx.incoming_metadata.iter().last().unwrap().memo.clone())).as_ref().map_or(false, |m| m.starts_with('{'))) .count() as u64 + 2 - }; + };*/ let incoming_metadata = IncomingTxMetadata { address: addr.clone(), value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1595,7 +1595,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1604,14 +1604,14 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }); txs.insert(tx.txid(), new_wtx); } info!("Successfully added txid with memo"); } else { - let position = 0; + // let position = 0; // Check if txid already exists in the hashmap let txid_exists = match self.txs.read() { @@ -1633,7 +1633,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1653,7 +1653,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1662,7 +1662,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, + // position: position, }); txs.insert(tx.txid(), new_wtx); } diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index 657f342..0034d1d 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -366,7 +366,7 @@ pub struct IncomingTxMetadata { pub value : u64, pub memo : Memo, pub incoming_mempool: bool, - pub position: u64, + // pub position: u64, } impl IncomingTxMetadata { @@ -378,7 +378,7 @@ impl IncomingTxMetadata { let value = reader.read_u64::()?; let incoming_mempool = true; - let position = 0; + // let position = 0; let mut memo_bytes = [0u8; 512]; reader.read_exact(&mut memo_bytes)?; @@ -389,7 +389,7 @@ impl IncomingTxMetadata { value, memo, incoming_mempool, - position, + // position, }) } From b0740177f341204d57a42b64dd22594236b1b22b Mon Sep 17 00:00:00 2001 From: lucretius Date: Tue, 23 Jan 2024 17:10:14 +0100 Subject: [PATCH 14/19] clean up --- lib/src/lightwallet.rs | 16 ---------------- lib/src/lightwallet/data.rs | 1 - 2 files changed, 17 deletions(-) diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 80a2c0c..e8f5622 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -1562,20 +1562,11 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 return; } - /*let position = if formatted_memo.as_ref().map_or(false, |m| m.starts_with('{')) { - 1 - } else { - existing_txs.iter() - .filter(|tx| !LightWallet::memo_str(&Some(tx.incoming_metadata.iter().last().unwrap().memo.clone())).as_ref().map_or(false, |m| m.starts_with('{'))) - .count() as u64 + 2 - };*/ - let incoming_metadata = IncomingTxMetadata { address: addr.clone(), value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1595,7 +1586,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1604,14 +1594,12 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }); txs.insert(tx.txid(), new_wtx); } info!("Successfully added txid with memo"); } else { - // let position = 0; // Check if txid already exists in the hashmap let txid_exists = match self.txs.read() { @@ -1633,7 +1621,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1653,7 +1640,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1662,7 +1648,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - // position: position, }); txs.insert(tx.txid(), new_wtx); } @@ -1685,7 +1670,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 } } - // Invalidate all blocks including and after "at_height". // Returns the number of blocks invalidated pub fn invalidate_block(&self, at_height: i32) -> u64 { diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index 0034d1d..5b0208e 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -366,7 +366,6 @@ pub struct IncomingTxMetadata { pub value : u64, pub memo : Memo, pub incoming_mempool: bool, - // pub position: u64, } impl IncomingTxMetadata { From c085c16328ae8a75a62f0547b134ffd49da59039 Mon Sep 17 00:00:00 2001 From: lucretius Date: Wed, 24 Jan 2024 07:04:12 +0100 Subject: [PATCH 15/19] rename total_fee to selected_fee, to better reflect the purpose --- lib/src/commands.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/commands.rs b/lib/src/commands.rs index 4f2b12a..007f4bb 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -554,16 +554,16 @@ impl Command for SendCommand { // If they can have different fees, you need to modify this logic. let default_fee: u64 = DEFAULT_FEE.try_into().unwrap(); - let mut total_fee = default_fee; + let mut selected_fee, = default_fee; for (_, _, _, fee) in send_args.iter() { if *fee != default_fee{ - total_fee = *fee; + selected_fee, = *fee; break; } } // Execute the transaction and handle the result. - match lightclient.do_send(tos, &total_fee) { + match lightclient.do_send(tos, &selected_fee,) { Ok(txid) => object!{ "txid" => txid }.pretty(2), Err(e) => object!{ "error" => e }.pretty(2), } From 4a954645f1574e3cfa5a7ba87c69861178d7eada Mon Sep 17 00:00:00 2001 From: lucretius Date: Wed, 24 Jan 2024 07:05:09 +0100 Subject: [PATCH 16/19] fix typo --- lib/src/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/commands.rs b/lib/src/commands.rs index 007f4bb..a8ae487 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -558,7 +558,7 @@ impl Command for SendCommand { for (_, _, _, fee) in send_args.iter() { if *fee != default_fee{ - selected_fee, = *fee; + selected_fee = *fee; break; } } From 1137ccd9a34ada41d4eb9b802cc4dc2e471b50ae Mon Sep 17 00:00:00 2001 From: lucretius Date: Wed, 24 Jan 2024 07:05:55 +0100 Subject: [PATCH 17/19] fix typo --- lib/src/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/commands.rs b/lib/src/commands.rs index a8ae487..5127803 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -554,7 +554,7 @@ impl Command for SendCommand { // If they can have different fees, you need to modify this logic. let default_fee: u64 = DEFAULT_FEE.try_into().unwrap(); - let mut selected_fee, = default_fee; + let mut selected_fee = default_fee; for (_, _, _, fee) in send_args.iter() { if *fee != default_fee{ From 81449bcfcd71ac979b0a0286ec48a2272362ab59 Mon Sep 17 00:00:00 2001 From: lucretius Date: Wed, 24 Jan 2024 07:07:39 +0100 Subject: [PATCH 18/19] copy pasta is bad, fixed superfluous , --- lib/src/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/commands.rs b/lib/src/commands.rs index 5127803..66f43be 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -563,7 +563,7 @@ impl Command for SendCommand { } } // Execute the transaction and handle the result. - match lightclient.do_send(tos, &selected_fee,) { + match lightclient.do_send(tos, &selected_fee) { Ok(txid) => object!{ "txid" => txid }.pretty(2), Err(e) => object!{ "error" => e }.pretty(2), } From 767929a59f30d9c2e540c06359760b49b6986766 Mon Sep 17 00:00:00 2001 From: onryo Date: Sat, 27 Jan 2024 21:58:40 +0100 Subject: [PATCH 19/19] Update checkpoints --- lib/src/lightclient/checkpoints.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/src/lightclient/checkpoints.rs b/lib/src/lightclient/checkpoints.rs index c1be8fe..ca29fbc 100644 --- a/lib/src/lightclient/checkpoints.rs +++ b/lib/src/lightclient/checkpoints.rs @@ -489,6 +489,15 @@ fn get_main_checkpoint(height: u64) -> Option<(u64, &'static str, &'static str) (1620000,"000000048c6667a8724512cbd999bc491ec8522b1f3817001c7ba485dec46d10", "01eb8a3b067a71d2e6686e825734082de9d83a0e95c9c263246addc16873547c68013e06723bb1d16eb6ec9877e1ad9cffcffc73699e9a2a9f88ad4f4b85ac3f0a681500014f6d4feac67da46c00c726a59ab226cfe1641b0dd6d19a87849f1b82d6838925000001bfaecc149cb1bb9d26b59b32c92bd73710003bfaeecee3da9c32c757954e0f38016a717ab8ceb7d7aa433073b2015bc8842f267aa66e2ccf620ae02b672a4eeb3201868aced3b0eeec75b2d06b6f97f035abeb83c8fb4657cea908d1e24707f89253000101990cb33ebc55fc786e97b85f0137bee6fc961f6c57f298cf6ed7c886269a5801e7178f4dde7b917f3ddd408bb11204ab309114a7d25c41530e93f328c98fcd5001afc9b45263c748c61fcf9b74c33073073a31a8fa20aad6215471d948344c50650000000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" ), +(1630000,"000000032c724b40a4fb6d3e613aed9f294f22b76fc30b6caaa3bfe7ce3b01ad", + "01bcfb44652794a52bcd36a8a474060a8b34657c099acfd3b01a911dd8708b6e060015000001f4977a26b4b404dd10fd25f6c1ca0b0e6ae6fda4e286ac271b74f173f524174701c7b1f83da227ced7c79dab2a7ed7d06722137f4c6ffcc6cbbee9fdd3d0cefe570001023a7ffe381530fc46c600ed9d0c541cebd1ffe150eaa582fec1cb662fe1740d01cba8683a2a6e1f077e71d9ddf45f5b68316b49a7bcb9c0c6710f5a22eb2678380001cf6d0cd85ec536a4ee1bd63d2f0b3583d737c5d34ee3f3bbefebc69a62b2b05e000000015a9a2c4b59accf35c7b050f27ad84a45d36c6ce22e587c3fa061860b4573a23a000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), +(1640000,"00000000d6605a8a213fa134a2bf74cc901a42d0706718090788d6915c7ebbfd", + "01c899797849266da90f3da00518b509acd1d8b08688ebc578317967558cf52768001501ceaec93c0f945afa93f65e67f07c01fdf1c1f7f582e501023dcf057703a5781901624168303aff13e03aeabed6f63ec50455cf0e13d3ef932a07018c79b221252f01cc2cfab26e394af78df175585e71e94ab58b3f4979e064030990d5cc642bee2f0101f758c67f56b64afa2acf815eee1fd4de4dd8dea365eb0388814cc63d13f25f01ef9a5bf6af257265aeb5faf4b391af67cafd29793a37323b49a7c14f3608b852000001a83ce7bdb63d8d8c2d39d957f7a920ebf22d8204b25f60061620086654891c460001053e5664b4b6e50c8fcf551f0a460abc2a54b780c4d26e3661aab9968729bb4001c02cad68a6613dcbccac64975fc6e7a2851654dd7fe4cae760bc507bf192034301fee6f9840f0913d91187507d7c76bc605a6b9aac4ff1e951990d1df6abbfe533015a9a2c4b59accf35c7b050f27ad84a45d36c6ce22e587c3fa061860b4573a23a000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), +(1650000,"00000000922fca1c2ab235f6592387433c0622665408caf1a689d588acd7ef4a", + "014a43866e55370ba3daa33649d975d748cc12c1ef3580cf0c93b424238fc2b2360015018ceaeea4244a4e58fb01160f29684fef2ec344bb2a7ddf852188720593627b5801659c17e9ef2b5e45b9e55d676c5ce5f9c8b193d41abefd02adc1fdf3372eef610001689605287033ba0551e500645d7b1a41d591ae4158ddc3671da0137da00bf86500000000012da2eae775f2bba0ff79df96b7759b5e93a03d02ca6c89603fa55e80215dda5b01ec4fc9be82884f1e6a324d5e8370b1b70714ea38235b90dec6b019f466f7ef5d0001b8dab7fd26bf4cae166a75905d75e88a40ce11773afe2b1363fc793df97934580001fefbd39896743ba6d44d4ac57a51044ed6384aec3c9c76b92131713918bfd0350001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), ]; find_checkpoint(height, checkpoints)