diff --git a/Cargo.lock b/Cargo.lock index 1317c15..fab19ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler32" version = "1.0.4" @@ -186,7 +188,7 @@ checksum = "9e0089c35ab7c6f2bc55ab23f769913f0ac65b1023e7e74638a1f43128dd5df2" [[package]] name = "bellman" version = "0.1.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bit-vec", "blake2s_simd", @@ -510,7 +512,7 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "ff" version = "0.4.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "byteorder", "ff_derive", @@ -520,7 +522,7 @@ dependencies = [ [[package]] name = "ff_derive" version = "0.3.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "num-bigint", "num-integer", @@ -678,7 +680,7 @@ dependencies = [ [[package]] name = "group" version = "0.1.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "ff", "rand 0.7.2", @@ -1158,7 +1160,7 @@ dependencies = [ [[package]] name = "pairing" version = "0.14.2" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "byteorder", "ff", @@ -2830,7 +2832,7 @@ dependencies = [ [[package]] name = "zcash_client_backend" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bech32", "bs58", @@ -2846,7 +2848,7 @@ dependencies = [ [[package]] name = "zcash_primitives" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "aes", "blake2b_simd", @@ -2869,7 +2871,7 @@ dependencies = [ [[package]] name = "zcash_proofs" version = "0.0.0" -source = "git+https://git.hush.is/hush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37#1a0204113d487cdaaf183c2967010e5214ff9e37" +source = "git+https://git.hush.is/hush/librustzcash.git?rev=acff1444ec373e9c3e37b47ca95bfd358e45255b#acff1444ec373e9c3e37b47ca95bfd358e45255b" dependencies = [ "bellman", "blake2b_simd", diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 95388cc..924d436 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -43,34 +43,34 @@ webpki-roots = "0.18.0" [dependencies.bellman] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false features = ["groth16"] [dependencies.pairing] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" [dependencies.zcash_client_backend] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false [dependencies.zcash_primitives] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false features = ["transparent-inputs"] [dependencies.zcash_proofs] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" default-features = false [dependencies.ff] git = "https://git.hush.is/hush/librustzcash.git" -rev= "1a0204113d487cdaaf183c2967010e5214ff9e37" +rev= "acff1444ec373e9c3e37b47ca95bfd358e45255b" features = ["ff_derive"] [build-dependencies] diff --git a/lib/proto/compact_formats.proto b/lib/proto/compact_formats.proto index 7e1bc54..cecce25 100644 --- a/lib/proto/compact_formats.proto +++ b/lib/proto/compact_formats.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package cash.z.wallet.sdk.rpc; -option go_package = "walletrpc"; +option go_package = "lightwalletd/walletrpc"; +option swift_prefix = ""; // Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. // bytes fields of hashes are in canonical little-endian format. @@ -11,11 +12,11 @@ option go_package = "walletrpc"; // 3. Update your witnesses to generate new Sapling spend proofs. message CompactBlock { uint32 protoVersion = 1; // the version of this wire format, for storage - uint64 height = 2; // the height of this block - bytes hash = 3; - bytes prevHash = 4; - uint32 time = 5; - bytes header = 6; // (hash, prevHash, and time) OR (full header) + uint64 height = 2; // the height of this block + bytes hash = 3; // the ID (hash) of this block, same as in block explorers + bytes prevHash = 4; // the ID (hash) of this block's predecessor + uint32 time = 5; // Unix epoch time when the block was mined + bytes header = 6; // (hash, prevHash, and time) OR (full header) repeated CompactTx vtx = 7; // compact transactions from this block } @@ -23,8 +24,8 @@ message CompactTx { // Index and hash will allow the receiver to call out to chain // explorers or other data structures to retrieve more information // about this transaction. - uint64 index = 1; - bytes hash = 2; + uint64 index = 1; // the index within the full block + bytes hash = 2; // the ID (hash) of this transaction, same as in block explorers // The transaction fee: present if server can provide. In the case of a // stateless server and a transaction with transparent inputs, this will be @@ -33,16 +34,32 @@ message CompactTx { // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) uint32 fee = 3; - repeated CompactSpend spends = 4; - repeated CompactOutput outputs = 5; + repeated CompactSaplingSpend spends = 4; + repeated CompactSaplingOutput outputs = 5; } +// CompactSaplingSpend is a Sapling Spend Description as described in 7.3 of the Zcash +// protocol specification. +message CompactSaplingSpend { + bytes nf = 1; // nullifier (see the Zcash protocol specification) +} + +// output is a Sapling Output Description as described in section 7.4 of the +// Zcash protocol spec. Total size is 948. +message CompactSaplingOutput { + bytes cmu = 1; // note commitment u-coordinate + bytes epk = 2; // ephemeral public key + bytes ciphertext = 3; // first 52 bytes of ciphertext +} + +/* message CompactSpend { - bytes nf = 1; + bytes nf = 1; // nullifier (see the Zcash protocol specification) } message CompactOutput { - bytes cmu = 1; - bytes epk = 2; - bytes ciphertext = 3; + bytes cmu = 1; // note commitment u-coordinate + bytes epk = 2; // ephemeral public key + bytes ciphertext = 3; // first 52 bytes of ciphertext } +*/ diff --git a/lib/proto/service.proto b/lib/proto/service.proto index e5719ad..aa52e39 100644 --- a/lib/proto/service.proto +++ b/lib/proto/service.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package cash.z.wallet.sdk.rpc; -option go_package = "walletrpc"; - +option go_package = "lightwalletd/walletrpc"; +option swift_prefix = ""; import "compact_formats.proto"; // A BlockID message contains identifiers to select a block: a height or a @@ -56,14 +56,13 @@ message LightdInfo { uint64 longestchain = 9; uint64 notarized = 10; } - message Coinsupply { string result = 1; string coin = 2; - int64 height = 3; - int64 supply = 4; - int64 zfunds = 5; - int64 total = 6; + uint64 height = 3; + uint64 supply = 4; + uint64 zfunds = 5; + uint64 total = 6; } message TransparentAddress { @@ -75,28 +74,97 @@ message TransparentAddressBlockFilter { BlockRange range = 2; } +message Address { + string address = 1; +} +message AddressList { + repeated string addresses = 1; +} +message Balance { + int64 valueZat = 1; +} + message Exclude { repeated bytes txid = 1; } +// The TreeState is derived from the Hush getblockmerkletree rpc. +// https://faq.hush.is/rpc/getblockmerkletree.html +message TreeState { + string network = 1; // "main" or "test" + uint64 height = 2; // block height + string hash = 3; // block id + uint32 time = 4; // Unix epoch time when the block was mined + string saplingTree = 5; // sapling commitment tree state +} + +// Results are sorted by height, which makes it easy to issue another +// request that picks up from where the previous left off. +message GetAddressUtxosArg { + repeated string addresses = 1; + uint64 startHeight = 2; + uint32 maxEntries = 3; // zero means unlimited +} +message GetAddressUtxosReply { + string address = 6; + bytes txid = 1; + int32 index = 2; + bytes script = 3; + int64 valueZat = 4; + uint64 height = 5; +} +message GetAddressUtxosReplyList { + repeated GetAddressUtxosReply addressUtxos = 1; +} + service CompactTxStreamer { - // Compact Blocks + // Return the height of the tip of the best chain rpc GetLatestBlock(ChainSpec) returns (BlockID) {} + // Return the compact block corresponding to the given block identifier rpc GetBlock(BlockID) returns (CompactBlock) {} + // Return a list of consecutive compact blocks rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {} - // Transactions + // Return the requested full (not compact) transaction (as from zcashd) rpc GetTransaction(TxFilter) returns (RawTransaction) {} + // Submit the given transaction to the Zcash network rpc SendTransaction(RawTransaction) returns (SendResponse) {} - // t-Address support + // Return the txids corresponding to the given t-address within the given block range + rpc GetTaddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} + // wrapper for GetTaddressTxids rpc GetAddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} + rpc GetTaddressBalance(AddressList) returns (Balance) {} + rpc GetTaddressBalanceStream(stream Address) returns (Balance) {} - // Misc - rpc GetLightdInfo(Empty) returns (LightdInfo) {} - rpc GetCoinsupply(Empty) returns (Coinsupply) {} - - //Mempool + // Return the compact transactions currently in the mempool; the results + // can be a few seconds out of date. If the Exclude list is empty, return + // all transactions; otherwise return all *except* those in the Exclude list + // (if any); this allows the client to avoid receiving transactions that it + // already has (from an earlier call to this rpc). The transaction IDs in the + // Exclude list can be shortened to any number of bytes to make the request + // more bandwidth-efficient; if two or more transactions in the mempool + // match a shortened txid, they are all sent (none is excluded). Transactions + // in the exclude list that don't exist in the mempool are ignored. rpc GetMempoolTx(Exclude) returns (stream CompactTx) {} + + // Return a stream of current Mempool transactions. This will keep the output stream open while + // there are mempool transactions. It will close the returned stream when a new block is mined. rpc GetMempoolStream(Empty) returns (stream RawTransaction) {} -} + + // GetTreeState returns the note commitment tree state corresponding to the given block. + // See section 3.7 of the Zcash protocol specification. It returns several other useful + // values also (even though they can be obtained using GetBlock). + // The block can be specified by either height or hash. + rpc GetTreeState(BlockID) returns (TreeState) {} + rpc GetLatestTreeState(Empty) returns (TreeState) {} + + rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {} + rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {} + + // Return information about this lightwalletd instance and the blockchain + rpc GetLightdInfo(Empty) returns (LightdInfo) {} + // Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) + // rpc Ping(Duration) returns (PingResponse) {} + rpc GetCoinsupply(Empty) returns (Coinsupply) {} +} \ No newline at end of file diff --git a/lib/src/commands.rs b/lib/src/commands.rs index b4835ad..66f43be 100644 --- a/lib/src/commands.rs +++ b/lib/src/commands.rs @@ -5,6 +5,8 @@ use json::{object}; use crate::lightclient::LightClient; use crate::lightwallet::LightWallet; +use zcash_primitives::transaction::components::amount::DEFAULT_FEE; +use std::convert::TryInto; pub trait Command { fn help(&self) -> String; @@ -489,69 +491,82 @@ impl Command for SendCommand { // Parse the args. There are two argument types. // 1 - A set of 2(+1 optional) arguments for a single address send representing address, value, memo? // 2 - A single argument in the form of a JSON string that is "[{address: address, value: value, memo: memo},...]" - // 1 - Destination address. T or Z address if args.len() < 1 || args.len() > 3 { return self.help(); } - // Check for a single argument that can be parsed as JSON let send_args = if args.len() == 1 { let arg_list = args[0]; - - let json_args = match json::parse(&arg_list) { - Ok(j) => j, - Err(e) => { - let es = format!("Couldn't understand JSON: {}", e); - return format!("{}\n{}", es, self.help()); - } - }; - - if !json_args.is_array() { - return format!("Couldn't parse argument as array\n{}", self.help()); + match json::parse(&arg_list) { + Ok(json_args) => { + // Check if the parsed JSON is an array. + if !json_args.is_array() { + return format!("Couldn't parse argument as array\n{}", self.help()); + } + + // Map each JSON object to a tuple (address, amount, memo, fee). + let maybe_send_args = json_args.members().map(|j| { + if !j.has_key("address") || !j.has_key("amount") { + Err(format!("Need 'address' and 'amount'\n")) + } else { + let fee = j["fee"].as_u64().unwrap_or(DEFAULT_FEE.try_into().unwrap()); + Ok(( + j["address"].as_str().unwrap().to_string(), + j["amount"].as_u64().unwrap(), + j["memo"].as_str().map(|s| s.to_string()), + fee + )) + } + }).collect::, u64)>, String>>(); + + // Handle any errors that occurred during mapping. + match maybe_send_args { + Ok(a) => a, + Err(s) => return format!("Error: {}\n{}", s, self.help()), + } + }, + Err(e) => return format!("Couldn't understand JSON: {}\n{}", e, self.help()), } - - let maybe_send_args = json_args.members().map( |j| { - if !j.has_key("address") || !j.has_key("amount") { - Err(format!("Need 'address' and 'amount'\n")) - } else { - Ok((j["address"].as_str().unwrap().to_string().clone(), j["amount"].as_u64().unwrap(), j["memo"].as_str().map(|s| s.to_string().clone()))) - } - }).collect::)>, String>>(); - - match maybe_send_args { - Ok(a) => a.clone(), - Err(s) => { return format!("Error: {}\n{}", s, self.help()); } - } - } else if args.len() == 2 || args.len() == 3 { + } else { + // Handle the case where individual arguments are provided. let address = args[0].to_string(); - - // Make sure we can parse the amount let value = match args[1].parse::() { Ok(amt) => amt, - Err(e) => { - return format!("Couldn't parse amount: {}", e); - } + Err(e) => return format!("Couldn't parse amount: {}", e), }; - - let memo = if args.len() == 3 { Some(args[2].to_string()) } else { None }; - - // Memo has to be None if not sending to a shileded address + let memo = args.get(2).map(|m| m.to_string()); + + // Memo should be None if the address is not shielded. if memo.is_some() && !LightWallet::is_shielded_address(&address, &lightclient.config) { return format!("Can't send a memo to the non-shielded address {}", address); } - - vec![(args[0].to_string(), value, memo)] - } else { - return self.help() + + // Create a vector with a single transaction (address, amount, memo). + vec![(address, value, memo, DEFAULT_FEE.try_into().unwrap())] }; + + // Transform transaction data into the required format (String -> &str). + let tos = send_args.iter().map(|(a, v, m, _)| (a.as_str(), *v, m.clone())).collect::>(); + + // Calculate the total fee for all transactions. + // This assumes that all transactions have the same fee. + // If they can have different fees, you need to modify this logic. + + let default_fee: u64 = DEFAULT_FEE.try_into().unwrap(); + let mut selected_fee = default_fee; - // Convert to the right format. String -> &str. - let tos = send_args.iter().map(|(a, v, m)| (a.as_str(), *v, m.clone()) ).collect::>(); - match lightclient.do_send(tos) { - Ok(txid) => { object!{ "txid" => txid } }, - Err(e) => { object!{ "error" => e } } - }.pretty(2) + for (_, _, _, fee) in send_args.iter() { + if *fee != default_fee{ + selected_fee = *fee; + break; + } +} + // Execute the transaction and handle the result. + match lightclient.do_send(tos, &selected_fee) { + Ok(txid) => object!{ "txid" => txid }.pretty(2), + Err(e) => object!{ "error" => e }.pretty(2), + } } } diff --git a/lib/src/grpcconnector.rs b/lib/src/grpcconnector.rs index 4419c0a..f7d9894 100644 --- a/lib/src/grpcconnector.rs +++ b/lib/src/grpcconnector.rs @@ -97,52 +97,64 @@ pub fn get_coinsupply(uri: http::Uri, no_cert: bool) -> Result(uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) - -> Result<(), Box> +async fn get_block_range( + uri: &http::Uri, + start_height: u64, + end_height: u64, + no_cert: bool, + pool: ThreadPool, + c: F +) -> Result<(), Box> where F : Fn(&[u8], u64) { let mut client = get_client(uri, no_cert).await?; - let bs = BlockId{ height: start_height, hash: vec!()}; - let be = BlockId{ height: end_height, hash: vec!()}; + let bs = BlockId { height: start_height, hash: vec![] }; + let be = BlockId { height: end_height, hash: vec![] }; - let request = Request::new(BlockRange{ start: Some(bs), end: Some(be) }); + let request = Request::new(BlockRange { start: Some(bs), end: Some(be) }); - // Channel where the blocks are sent. A None signifies end of all blocks - let (tx, rx) = channel::>(); - - // Channel that the processor signals it is done, so the method can return - let (ftx, frx) = channel(); - - // The processor runs on a different thread, so that the network calls don't - // block on this - pool.execute(move || { - while let Some(block) = rx.recv().unwrap() { - use prost::Message; - let mut encoded_buf = vec![]; - - block.encode(&mut encoded_buf).unwrap(); - c(&encoded_buf, block.height); - } - - ftx.send(Ok(())).unwrap(); - }); + let (tx, rx) = channel::>(); + let (ftx, frx) = channel(); + pool.execute(move || { + while let Ok(Some(block)) = rx.recv() { + use prost::Message; + let mut encoded_buf = vec![]; + + if let Err(e) = block.encode(&mut encoded_buf) { + eprintln!("Error encoding block: {:?}", e); + break; + } + + c(&encoded_buf, block.height); + } + + if let Err(e) = ftx.send(Ok(())) { + eprintln!("Error sending completion signal: {:?}", e); + } + }); + let mut response = client.get_block_range(request).await?.into_inner(); - //println!("{:?}", response); - while let Some(block) = response.message().await? { - tx.send(Some(block)).unwrap(); - } - tx.send(None).unwrap(); - // Wait for the processor to exit + while let Some(block) = response.message().await? { + if let Err(e) = tx.send(Some(block)) { + eprintln!("Error sending block to channel: {:?}", e); + break; + } + } + + if let Err(e) = tx.send(None) { + eprintln!("Error sending end signal to channel: {:?}", e); + } + frx.iter().take(1).collect::, String>>()?; Ok(()) } + pub fn fetch_blocks(uri: &http::Uri, start_height: u64, end_height: u64, no_cert: bool, pool: ThreadPool, c: F) -> Result<(), String> where F : Fn(&[u8], u64) { @@ -151,7 +163,6 @@ pub fn fetch_blocks(uri: &http::Uri, start_heig Err(e) => { let es = format!("Error creating runtime {:?}", e); error!("{}", es); - eprintln!("{}", e); return Err(es); } }; @@ -161,31 +172,49 @@ pub fn fetch_blocks(uri: &http::Uri, start_heig Err(e) => { let e = format!("Error fetching blocks {:?}", e); error!("{}", e); - eprintln!("{}", e); Err(e) } } } - // get_address_txids GRPC call -async fn get_address_txids(uri: &http::Uri, address: String, - start_height: u64, end_height: u64, no_cert: bool, c: F) -> Result<(), Box> - where F : Fn(&[u8], u64) { +async fn get_address_txids( + uri: &http::Uri, + address: String, + start_height: u64, + end_height: u64, + no_cert: bool, + c: F +) -> Result<(), Box> +where F : Fn(&[u8], u64) { + + let mut client = match get_client(uri, no_cert).await { + Ok(client) => client, + Err(e) => { + eprintln!("Error creating client: {:?}", e); + return Err(e.into()); + } + }; - let mut client = get_client(uri, no_cert).await?; let start = Some(BlockId{ height: start_height, hash: vec!()}); - let end = Some(BlockId{ height: end_height, hash: vec!()}); + let end = Some(BlockId{ height: end_height, hash: vec!()}); - let request = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{start, end}) }); + let request = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{ start, end }) }); + + let maybe_response = match client.get_address_txids(request).await { + Ok(response) => response, + Err(e) => { + eprintln!("Error getting address txids: {:?}", e); + return Err(e.into()); + } + }; - let maybe_response = client.get_address_txids(request).await?; let mut response = maybe_response.into_inner(); while let Some(tx) = response.message().await? { c(&tx.data, tx.height); } - + Ok(()) } @@ -223,16 +252,21 @@ where Ok(()) } -pub fn fetch_transparent_txids(uri: &http::Uri, address: String, - start_height: u64, end_height: u64, no_cert: bool, c: F) -> Result<(), String> - where F : Fn(&[u8], u64) { - +pub fn fetch_transparent_txids( + uri: &http::Uri, + address: String, + start_height: u64, + end_height: u64, + no_cert: bool, + c: F +) -> Result<(), String> +where F : Fn(&[u8], u64) { + let mut rt = match tokio::runtime::Runtime::new() { Ok(r) => r, Err(e) => { let e = format!("Error creating runtime {:?}", e); error!("{}", e); - eprintln!("{}", e); return Err(e); } }; @@ -242,13 +276,11 @@ pub fn fetch_transparent_txids(uri: &http::Uri, Err(e) => { let e = format!("Error with get_address_txids runtime {:?}", e); error!("{}", e); - eprintln!("{}", e); - Err(e) + return Err(e) } } } - // get_transaction GRPC call async fn get_transaction(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result> { @@ -266,7 +298,6 @@ pub fn fetch_full_tx(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result { let errstr = format!("Error creating runtime {}", e.to_string()); error!("{}", errstr); - eprintln!("{}", errstr); return Err(errstr); } }; @@ -276,7 +307,6 @@ pub fn fetch_full_tx(uri: &http::Uri, txid: TxId, no_cert: bool) -> Result { let errstr = format!("Error in get_transaction runtime {}", e.to_string()); error!("{}", errstr); - eprintln!("{}", errstr); Err(errstr) } } diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index 4f53169..06f004e 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -953,11 +953,11 @@ impl LightClient { wtxs.iter().flat_map(|wtx| { wtx.incoming_metadata.iter() .enumerate() - .map(move |(_i, om)| + .map(move |(i, om)| object! { "block_height" => wtx.block.clone(), "datetime" => wtx.datetime.clone(), - "position" => om.position, + "position" => i, "txid" => format!("{}", wtx.txid), "amount" => om.value as i64, "address" => om.address.clone(), @@ -1233,7 +1233,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } // Sleep exponentially backing off std::thread::sleep(std::time::Duration::from_secs((2 as u64).pow(retry_count))); - println!("Sync error {}\nRetry count {}", e, retry_count); + // println!("Sync error {}\nRetry count {}", e, retry_count); } } } @@ -1252,6 +1252,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } let addr = address.or(self.wallet.read().unwrap().get_all_zaddresses().get(0).map(|s| s.clone())).unwrap(); + let fee: u64 = DEFAULT_FEE.try_into().unwrap(); let result = { let _lock = self.sync_lock.lock().unwrap(); @@ -1260,6 +1261,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { &self.sapling_spend, &self.sapling_output, true, vec![(&addr, tbal - fee, None)], + &fee, |txbytes| broadcast_raw_tx(&self.get_server_uri(),self.config.no_cert_verification, txbytes) ) }; @@ -1477,47 +1479,83 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { } } } - - fn scan_taddress_txids(&self, pool: &ThreadPool, block_times: Arc>>, start_height: u64, end_height: u64, no_cert: bool) -> Result, String> { - // Copy over addresses so as to not lock up the wallet, which we'll use inside the callback below. - let addresses = self.wallet.read().unwrap() + fn scan_taddress_txids( + &self, + pool: &ThreadPool, + block_times: Arc>>, + start_height: u64, + end_height: u64, + no_cert: bool + ) -> Result, String> { + let addresses = self.wallet.read() + .map_err(|e| format!("Failed to read wallet: {:?}", e))? .get_all_taddresses().iter() - .map(|a| a.clone()) + .cloned() .collect::>(); - - // Create a channel so the fetch_transparent_txids can send the results back + let (ctx, crx) = channel(); let num_addresses = addresses.len(); - + for address in addresses { + let address_clone = address.clone(); let wallet = self.wallet.clone(); - let pool = pool.clone(); let server_uri = self.get_server_uri(); let ctx = ctx.clone(); - let block_times = block_times.clone(); - + pool.execute(move || { - // Fetch the transparent transactions for this address, and send the results - // via the channel - let r = fetch_transparent_txids(&server_uri, address, start_height, end_height,no_cert, - move |tx_bytes: &[u8], height: u64| { - let tx = Transaction::read(tx_bytes).unwrap(); - - // Scan this Tx for transparent inputs and outputs - let datetime = block_times.read().unwrap().get(&height).map(|v| *v).unwrap_or(0); - wallet.read().unwrap().scan_full_tx(&tx, height as i32, datetime as u64); - }); - ctx.send(r).unwrap(); + + let r = fetch_transparent_txids( + &server_uri, + address, + start_height, + end_height, + no_cert, + move |tx_bytes: &[u8], height: u64| { + let tx_result = Transaction::read(tx_bytes) + .map_err(|e| format!("Failed to read transaction: {:?}", e)); + + match tx_result { + Ok(tx) => { + let datetime_result = block_times.read() + .map_err(|e| format!("Failed to read block times: {:?}", e)) + .and_then(|bt| bt.get(&height).cloned().ok_or_else(|| format!("No datetime for height: {}", height))); + + match datetime_result { + Ok(datetime) => { + match wallet.read().map_err(|e| format!("Failed to read wallet: {:?}", e)) { + Ok(w) => { + w.scan_full_tx(&tx, height as i32, datetime as u64); + }, + Err(e) => { + println!("Error reading wallet: {}", e); + }, + } + }, + Err(e) => { + println!("Error processing transaction: {}", e); + }, + } + }, + Err(e) => { + println!("Error reading transaction: {}", e); + }, + } + } + ); + + match ctx.send(r) { + Ok(_) => info!("Successfully sent data for address: {}", address_clone), + Err(e) => println!("Failed to send data for address: {}: {:?}", address_clone, e), + } }); } - - // Collect all results from the transparent fetches, and make sure everything was OK. - // If it was not, we return an error, which will go back to the retry - crx.iter().take(num_addresses).collect::, String>>() - } - + + crx.iter().take(num_addresses).collect() + } + + fn scan_fill_fulltxs(&self, pool: &ThreadPool, decoy_txids: Vec<(TxId, i32)>) -> Result, String> { // We need to first copy over the Txids from the wallet struct, because @@ -1564,7 +1602,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { crx.iter().take(num_fetches).collect::, String>>() } - pub fn do_send(&self, addrs: Vec<(&str, u64, Option)>) -> Result { + pub fn do_send(&self, addrs: Vec<(&str, u64, Option)>, fee: &u64) -> Result { if !self.wallet.read().unwrap().is_unlocked_for_spending() { error!("Wallet is locked"); return Err("Wallet is locked".to_string()); @@ -1580,6 +1618,7 @@ pub fn start_mempool_monitor(lc: Arc) -> Result<(), String> { &self.sapling_spend, &self.sapling_output, false, addrs, + fee, |txbytes| broadcast_raw_tx(&self.get_server_uri(), self.config.no_cert_verification, txbytes) ) }; diff --git a/lib/src/lightclient/checkpoints.rs b/lib/src/lightclient/checkpoints.rs index c1be8fe..ca29fbc 100644 --- a/lib/src/lightclient/checkpoints.rs +++ b/lib/src/lightclient/checkpoints.rs @@ -489,6 +489,15 @@ fn get_main_checkpoint(height: u64) -> Option<(u64, &'static str, &'static str) (1620000,"000000048c6667a8724512cbd999bc491ec8522b1f3817001c7ba485dec46d10", "01eb8a3b067a71d2e6686e825734082de9d83a0e95c9c263246addc16873547c68013e06723bb1d16eb6ec9877e1ad9cffcffc73699e9a2a9f88ad4f4b85ac3f0a681500014f6d4feac67da46c00c726a59ab226cfe1641b0dd6d19a87849f1b82d6838925000001bfaecc149cb1bb9d26b59b32c92bd73710003bfaeecee3da9c32c757954e0f38016a717ab8ceb7d7aa433073b2015bc8842f267aa66e2ccf620ae02b672a4eeb3201868aced3b0eeec75b2d06b6f97f035abeb83c8fb4657cea908d1e24707f89253000101990cb33ebc55fc786e97b85f0137bee6fc961f6c57f298cf6ed7c886269a5801e7178f4dde7b917f3ddd408bb11204ab309114a7d25c41530e93f328c98fcd5001afc9b45263c748c61fcf9b74c33073073a31a8fa20aad6215471d948344c50650000000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" ), +(1630000,"000000032c724b40a4fb6d3e613aed9f294f22b76fc30b6caaa3bfe7ce3b01ad", + "01bcfb44652794a52bcd36a8a474060a8b34657c099acfd3b01a911dd8708b6e060015000001f4977a26b4b404dd10fd25f6c1ca0b0e6ae6fda4e286ac271b74f173f524174701c7b1f83da227ced7c79dab2a7ed7d06722137f4c6ffcc6cbbee9fdd3d0cefe570001023a7ffe381530fc46c600ed9d0c541cebd1ffe150eaa582fec1cb662fe1740d01cba8683a2a6e1f077e71d9ddf45f5b68316b49a7bcb9c0c6710f5a22eb2678380001cf6d0cd85ec536a4ee1bd63d2f0b3583d737c5d34ee3f3bbefebc69a62b2b05e000000015a9a2c4b59accf35c7b050f27ad84a45d36c6ce22e587c3fa061860b4573a23a000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), +(1640000,"00000000d6605a8a213fa134a2bf74cc901a42d0706718090788d6915c7ebbfd", + "01c899797849266da90f3da00518b509acd1d8b08688ebc578317967558cf52768001501ceaec93c0f945afa93f65e67f07c01fdf1c1f7f582e501023dcf057703a5781901624168303aff13e03aeabed6f63ec50455cf0e13d3ef932a07018c79b221252f01cc2cfab26e394af78df175585e71e94ab58b3f4979e064030990d5cc642bee2f0101f758c67f56b64afa2acf815eee1fd4de4dd8dea365eb0388814cc63d13f25f01ef9a5bf6af257265aeb5faf4b391af67cafd29793a37323b49a7c14f3608b852000001a83ce7bdb63d8d8c2d39d957f7a920ebf22d8204b25f60061620086654891c460001053e5664b4b6e50c8fcf551f0a460abc2a54b780c4d26e3661aab9968729bb4001c02cad68a6613dcbccac64975fc6e7a2851654dd7fe4cae760bc507bf192034301fee6f9840f0913d91187507d7c76bc605a6b9aac4ff1e951990d1df6abbfe533015a9a2c4b59accf35c7b050f27ad84a45d36c6ce22e587c3fa061860b4573a23a000001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), +(1650000,"00000000922fca1c2ab235f6592387433c0622665408caf1a689d588acd7ef4a", + "014a43866e55370ba3daa33649d975d748cc12c1ef3580cf0c93b424238fc2b2360015018ceaeea4244a4e58fb01160f29684fef2ec344bb2a7ddf852188720593627b5801659c17e9ef2b5e45b9e55d676c5ce5f9c8b193d41abefd02adc1fdf3372eef610001689605287033ba0551e500645d7b1a41d591ae4158ddc3671da0137da00bf86500000000012da2eae775f2bba0ff79df96b7759b5e93a03d02ca6c89603fa55e80215dda5b01ec4fc9be82884f1e6a324d5e8370b1b70714ea38235b90dec6b019f466f7ef5d0001b8dab7fd26bf4cae166a75905d75e88a40ce11773afe2b1363fc793df97934580001fefbd39896743ba6d44d4ac57a51044ed6384aec3c9c76b92131713918bfd0350001c2b480770a4ed4aa312595eba91cb25a5e4cb4dd368b51f0d639f866a23fe12e000001491139c6c100cdd9176607c63fa695709727d919634d2983a9412c3010ed6d3d01708c9850eb440b259f233187662c5228804cb4500263949301b6fac8f6428f2301d6f84c424acdb1d10f8cef641662e0f63f954f07fe6199d504a61979c9ba3e13" +), ]; find_checkpoint(height, checkpoints) diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 3271a95..e8f5622 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -39,7 +39,7 @@ use zcash_primitives::{ serialize::{Vector}, transaction::{ builder::{Builder}, - components::{Amount, OutPoint, TxOut}, components::amount::DEFAULT_FEE, + components::{Amount, OutPoint, TxOut}, TxId, Transaction, }, sapling::Node, @@ -1470,26 +1470,43 @@ pub fn scan_full_tx(&self, tx: &Transaction, height: i32, datetime: u64) { // Do it in a short scope because of the write lock. { info!("A sapling output was sent in {}", tx.txid()); - let mut txs = self.txs.write().unwrap(); - if txs.get(&tx.txid()).unwrap().outgoing_metadata.iter() - .find(|om| om.address == address && om.value == note.value && om.memo == memo) - .is_some() { - warn!("Duplicate outgoing metadata"); - continue; - } + match self.txs.write() { + Ok(mut txs) => { + match txs.get(&tx.txid()) { + Some(wtx) => { + if wtx.outgoing_metadata.iter() + .any(|om| om.address == address && om.value == note.value && om.memo == memo) + { + warn!("Duplicate outgoing metadata"); + continue; + } - // Write the outgoing metadata - txs.get_mut(&tx.txid()).unwrap() - .outgoing_metadata - .push(OutgoingTxMetadata{ - address, value: note.value, memo, - }); + // Write the outgoing metadata + txs.get_mut(&tx.txid()).unwrap() + .outgoing_metadata + .push(OutgoingTxMetadata { + address, + value: note.value, + memo, + }); + }, + None => { + error!("Can not find any entry for txid : {}", tx.txid()); + continue; + } + } + }, + Err(poisoned) => { + error!("Lock is poisoned: {}", poisoned); + return; + } + } } }, None => {} }; } - } + } // Mark this Tx as scanned { let mut txs = self.txs.write().unwrap(); @@ -1545,20 +1562,11 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 return; } - let position = if formatted_memo.as_ref().map_or(false, |m| m.starts_with('{')) { - 1 - } else { - existing_txs.iter() - .filter(|tx| !LightWallet::memo_str(&Some(tx.incoming_metadata.iter().last().unwrap().memo.clone())).as_ref().map_or(false, |m| m.starts_with('{'))) - .count() as u64 + 2 - }; - let incoming_metadata = IncomingTxMetadata { address: addr.clone(), value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1578,7 +1586,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1587,14 +1594,12 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }); txs.insert(tx.txid(), new_wtx); } info!("Successfully added txid with memo"); } else { - let position = 0; // Check if txid already exists in the hashmap let txid_exists = match self.txs.read() { @@ -1616,7 +1621,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }; wtx.incoming_metadata.push(incoming_metadata); @@ -1636,7 +1640,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }); } else { let mut new_wtx = WalletTx::new(height, now() as u64, &tx.txid()); @@ -1645,7 +1648,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 value: amt, memo: memo.clone(), incoming_mempool: true, - position: position, }); txs.insert(tx.txid(), new_wtx); } @@ -1668,7 +1670,6 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 } } - // Invalidate all blocks including and after "at_height". // Returns the number of blocks invalidated pub fn invalidate_block(&self, at_height: i32) -> u64 { @@ -2177,6 +2178,7 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 output_params: &[u8], _transparent_only: bool, tos: Vec<(&str, u64, Option)>, + fee: &u64, broadcast_fn: F ) -> Result<(String, Vec), String> where F: Fn(Box<[u8]>) -> Result @@ -2207,24 +2209,31 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 total_value, tos.len() ); - // Convert address (str) to RecepientAddress and value to Amount - let recepients = tos.iter().map(|to| { - let ra = match address::RecipientAddress::from_str(to.0, - self.config.hrp_sapling_address(), - self.config.base58_pubkey_address(), - self.config.base58_script_address()) { - Some(to) => to, - None => { - let e = format!("Invalid recipient address: '{}'", to.0); - error!("{}", e); - return Err(e); - } - }; + // Convert address (str) to RecipientAddress and value to Amount - let value = Amount::from_u64(to.1).unwrap(); + let recepients: Result)>, String> = tos.iter().map(|to| { + // Convert string to RecipientAddress + let ra = match address::RecipientAddress::from_str( + to.0, + self.config.hrp_sapling_address(), + self.config.base58_pubkey_address(), + self.config.base58_script_address() + ) { + Some(addr) => addr, + None => { + let e = format!("Invalid recipient address: '{}'", to.0); + error!("{}", e); + return Err(e); + } + }; - Ok((ra, value, to.2.clone())) - }).collect::)>, String>>()?; + // Convert the second tuple element to Amount + let value = Amount::from_u64(to.1).expect("Invalid amount value"); + + Ok((ra, value, to.2.clone())) + }).collect(); + + let recepients = recepients?; // Target the next block, assuming we are up-to-date. let (height, anchor_offset) = match self.get_target_height_and_anchor_offset() { @@ -2237,8 +2246,9 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 }; // Select notes to cover the target value - println!("{}: Selecting notes", now() - start_time); - let target_value = Amount::from_u64(total_value).unwrap() + DEFAULT_FEE ; + // Select notes to cover the target value + println!("{}: Selecting notes", now() - start_time); + let target_value = Amount::from_u64(total_value).unwrap() + Amount::from_u64(*fee).unwrap(); // Select the candidate notes that are eligible to be spent let notes: Vec<_> = self.txs.read().unwrap().iter() .map(|(txid, tx)| tx.notes.iter().map(move |note| (*txid, note))) @@ -2323,8 +2333,11 @@ pub fn scan_full_mempool_tx(&self, tx: &Transaction, height: i32, _datetime: u64 return Err(e); } + let fee_amount = Amount::from_u64(*fee).expect("Invalid fee amount"); + builder.set_fee(fee_amount); + // Create the transaction - println!("{}: Adding {} notes and {} utxos", now() - start_time, notes.len(), tinputs.len()); + println!("{}: Adding {} notes and {} utxos and fee {:?}", now() - start_time, notes.len(), tinputs.len(), fee_amount); for selected in notes.iter() { if let Err(e) = builder.add_sapling_spend( diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index eb57b13..5b0208e 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -366,7 +366,6 @@ pub struct IncomingTxMetadata { pub value : u64, pub memo : Memo, pub incoming_mempool: bool, - pub position: u64, } impl IncomingTxMetadata { @@ -378,7 +377,7 @@ impl IncomingTxMetadata { let value = reader.read_u64::()?; let incoming_mempool = true; - let position = 0; + // let position = 0; let mut memo_bytes = [0u8; 512]; reader.read_exact(&mut memo_bytes)?; @@ -389,7 +388,7 @@ impl IncomingTxMetadata { value, memo, incoming_mempool, - position, + // position, }) } @@ -440,7 +439,7 @@ pub struct WalletTx { impl WalletTx { pub fn serialized_version() -> u64 { - return 4; + return 5; } pub fn new(height: i32, datetime: u64, txid: &TxId) -> Self { @@ -460,10 +459,9 @@ impl WalletTx { pub fn read(mut reader: R) -> io::Result { let version = reader.read_u64::()?; - assert!(version <= WalletTx::serialized_version()); + assert!(version <= WalletTx::serialized_version(), "Version mismatch. Please restore with your Seed"); let block = reader.read_i32::()?; - let datetime = if version >= 4 { reader.read_u64::()? } else { @@ -472,23 +470,24 @@ impl WalletTx { let mut txid_bytes = [0u8; 32]; reader.read_exact(&mut txid_bytes)?; - let txid = TxId{0: txid_bytes}; let notes = Vector::read(&mut reader, |r| SaplingNoteData::read(r))?; let utxos = Vector::read(&mut reader, |r| Utxo::read(r))?; - let total_shielded_value_spent = reader.read_u64::()?; let total_transparent_value_spent = reader.read_u64::()?; - - // Outgoing metadata was only added in version 2 let outgoing_metadata = Vector::read(&mut reader, |r| OutgoingTxMetadata::read(r))?; - let incoming_metadata = Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))?; + // Read incoming_metadata only if version is 5 or higher + let incoming_metadata = if version >= 5 { + Vector::read(&mut reader, |r| IncomingTxMetadata::read(r))? + } else { + vec![] + }; let full_tx_scanned = reader.read_u8()? > 0; - - Ok(WalletTx{ + + Ok(WalletTx { block, datetime, txid, @@ -502,6 +501,7 @@ impl WalletTx { }) } + pub fn write(&self, mut writer: W) -> io::Result<()> { writer.write_u64::(WalletTx::serialized_version())?;