Merge pull request #21 from DenioD/master

Add Checkpoint and some upstream improvements

Cargo.lock

@@ -1724,7 +1724,7 @@ dependencies = [
 
 [[package]]
 name = "silentdragonlite-cli"
-version = "1.1.0"
+version = "1.3.2"
 dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1752,6 +1752,7 @@ dependencies = [
  "http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "json 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libflate 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "log4rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "pairing 0.14.2 (git+https://github.com/MyHush/librustzcash.git?rev=1a0204113d487cdaaf183c2967010e5214ff9e37)",

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "silentdragonlite-cli"
-version = "1.1.0"
+version = "1.3.2"
 edition = "2018"
 
 [dependencies]

cli/src/lib.rs

@@ -11,7 +11,7 @@ use silentdragonlitelib::{commands,
 #[macro_export]
 macro_rules! configure_clapapp {
     ( $freshapp: expr ) => {
-    $freshapp.version("1.0.0")
+        $freshapp.version("1.3.2")
             .arg(Arg::with_name("dangerous")
                 .long("dangerous")
                 .help("Disable server TLS certificate verification. Use this if you're running a local lightwalletd with a self-signed certificate. WARNING: This is dangerous, don't use it with a server that is not your own.")
@@ -25,6 +25,10 @@ macro_rules! configure_clapapp {
                 .long("recover")
                 .help("Attempt to recover the seed from the wallet")
                 .takes_value(false))
+                .arg(Arg::with_name("password")
+                .long("password")
+                .help("When recovering seed, specify a password for the encrypted wallet")
+                .takes_value(true))
             .arg(Arg::with_name("seed")
                 .short("s")
                 .long("seed")
@@ -232,7 +236,7 @@ pub fn command_loop(lightclient: Arc<LightClient>) -> (Sender<(String, Vec<Strin
     (command_tx, resp_rx)
 }
 
-pub fn attempt_recover_seed() {
+pub fn attempt_recover_seed(password: Option<String>) {
     // Create a Light Client Config in an attempt to recover the file.
     let config = LightClientConfig {
         server: "0.0.0.0:0".parse().unwrap(),
@@ -244,7 +248,7 @@ pub fn attempt_recover_seed() {
         data_dir: None,
     };
 
-    match LightClient::attempt_recover_seed(&config) {
+    match LightClient::attempt_recover_seed(&config, password) {
         Ok(seed) => println!("Recovered seed: '{}'", seed),
         Err(e)   => eprintln!("Failed to recover seed. Error: {}", e)
     };

cli/src/main.rs

@@ -4,6 +4,7 @@ use silentdragonlite_cli::{configure_clapapp,
                     startup,
                     start_interactive,
                     attempt_recover_seed};
+                    //version::VERSION
 use log::error;
 
 pub fn main() {
@@ -12,9 +13,10 @@ pub fn main() {
     let fresh_app = App::new("SilentDragonLite CLI");
     let configured_app = configure_clapapp!(fresh_app);
     let matches = configured_app.get_matches();
+
     if matches.is_present("recover") {
         // Create a Light Client Config in an attempt to recover the file.
-        attempt_recover_seed();
+        attempt_recover_seed(matches.value_of("password").map(|s| s.to_string()));
         return;
     }
 
@@ -54,8 +56,9 @@ pub fn main() {
     let (command_tx, resp_rx) = match startup(server, dangerous, seed, birthday, !nosync, command.is_none()) {
         Ok(c) => c,
         Err(e) => {
-            eprintln!("Error during startup: {}", e);
+            let emsg = format!("Error during startup:{}\nIf you repeatedly run into this issue, you might have to restore your wallet from your seed phrase.", e);
-            error!("Error during startup: {}", e);
+            eprintln!("{}", emsg);
+            error!("{}", emsg);
             if cfg!(target_os = "unix" ) {
                 match e.raw_os_error() {
                     Some(13) => report_permission_error(),

cli/src/version.rs

@@ -0,0 +1 @@
+pub const VERSION:&str = "1.3.2";

lib/Cargo.toml

@@ -22,6 +22,7 @@ rust-embed = { version = "5.1.0", features = ["debug-embed"] }
 rand = "0.7.2"
 sodiumoxide = "0.2.5"
 ring = "0.16.9"
+libflate = "0.1"
 
 tonic = { version = "0.1.1", features = ["tls", "tls-roots"] }
 bytes = "0.4"

lib/src/commands.rs

@@ -241,10 +241,7 @@ impl Command for BalanceCommand {
     }
 
     fn exec(&self, _args: &[&str], lightclient: &LightClient) -> String {
-        match lightclient.do_sync(true) {
+        format!("{}", lightclient.do_balance().pretty(2))
-            Ok(_) => format!("{}", lightclient.do_balance().pretty(2)),
-            Err(e) => e
-        }
     }
 }
 
@@ -649,12 +646,7 @@ impl Command for TransactionsCommand {
     }
 
     fn exec(&self, _args: &[&str], lightclient: &LightClient) -> String {
-        match lightclient.do_sync(true) {
+        format!("{}", lightclient.do_list_transactions().pretty(2))
-            Ok(_) => {
-                format!("{}", lightclient.do_list_transactions().pretty(2))
-            },
-            Err(e) => e
-        }
     }
 }
 
@@ -680,14 +672,7 @@ impl Command for HeightCommand {
             return format!("Didn't understand arguments\n{}", self.help());
         }
 
-        if args.len() == 0 || (args.len() == 1 && args[0].trim() == "true") {
+        format!("{}", object! { "height" => lightclient.last_scanned_height()}.pretty(2))
-            match lightclient.do_sync(true) {
-                Ok(_) => format!("{}", object! { "height" => lightclient.last_scanned_height()}.pretty(2)),
-                Err(e) => e
-            }
-        } else {
-            format!("{}", object! { "height" => lightclient.last_scanned_height()}.pretty(2))
-        }
     }
 }
 
@@ -785,12 +770,7 @@ impl Command for NotesCommand {
             false
         };
 
-        match lightclient.do_sync(true) {
+        format!("{}", lightclient.do_list_notes(all_notes).pretty(2))
-            Ok(_) => {
-                format!("{}", lightclient.do_list_notes(all_notes).pretty(2))
-            },
-            Err(e) => e
-        }
     }
 }
 

lib/src/lightclient.rs

@@ -342,6 +342,7 @@ impl LightClient {
 
         info!("Created new wallet with a new seed!");
         info!("Created LightClient to {}", &config.server);
+        l.do_save().map_err(|s| io::Error::new(ErrorKind::PermissionDenied, s))?;
 
         Ok(l)
     }
@@ -367,6 +368,7 @@ impl LightClient {
 
         info!("Created new wallet!");
         info!("Created LightClient to {}", &config.server);
+        l.do_save().map_err(|s| io::Error::new(ErrorKind::PermissionDenied, s))?;
 
         Ok(l)
     }
@@ -413,24 +415,32 @@ impl LightClient {
         Ok(())
     }
 
-    pub fn attempt_recover_seed(config: &LightClientConfig) -> Result<String, String> {
+        pub fn attempt_recover_seed(config: &LightClientConfig, password: Option<String>) -> Result<String, String> {
         use std::io::prelude::*;
-        use byteorder::{LittleEndian, ReadBytesExt,};
+        use byteorder::{LittleEndian, ReadBytesExt};
+        use libflate::gzip::Decoder;
         use bip39::{Mnemonic, Language};
         use zcash_primitives::serialize::Vector;
 
-        let mut reader = BufReader::new(File::open(config.get_wallet_path()).unwrap());
+        let mut inp = BufReader::new(File::open(config.get_wallet_path()).unwrap());
-        let version = reader.read_u64::<LittleEndian>().unwrap();
+        let version = inp.read_u64::<LittleEndian>().unwrap();
         println!("Reading wallet version {}", version);
 
+        // After version 5, we're writing the rest of the file as a compressed stream (gzip)
+        let mut reader: Box<dyn Read> = if version <= 4 {
+            Box::new(inp)
+        } else {
+            Box::new(Decoder::new(inp).unwrap())
+        };
+
         let encrypted = if version >= 4 {
             reader.read_u8().unwrap() > 0
         } else {
             false
         };
 
-        if encrypted {
+        if encrypted && password.is_none() {
-            return Err("The wallet is encrypted!".to_string());
+            return Err("The wallet is encrypted and a password was not specified. Please specify the password with '--password'!".to_string());
         }
 
         let mut enc_seed = [0u8; 48];
@@ -438,19 +448,35 @@ impl LightClient {
             reader.read_exact(&mut enc_seed).unwrap();
         }
 
-        let _nonce = if version >= 4 {
+        let nonce = if version >= 4 {
             Vector::read(&mut reader, |r| r.read_u8()).unwrap()
         } else {
             vec![]
         };
 
+        let phrase = if encrypted {
+            use sodiumoxide::crypto::secretbox;
+            use crate::lightwallet::double_sha256;
+
+         // Get the doublesha256 of the password, which is the right length
+         let key = secretbox::Key::from_slice(&double_sha256(password.unwrap().as_bytes())).unwrap();
+         let nonce = secretbox::Nonce::from_slice(&nonce).unwrap();
+
+         let seed = match secretbox::open(&enc_seed, &nonce, &key) {
+            Ok(s) => s,
+            Err(_) => return Err("Decryption failed. Is your password correct?".to_string())
+        };
+
+        Mnemonic::from_entropy(&seed, Language::English)
+    } else {
         // Seed
         let mut seed_bytes = [0u8; 32];
         reader.read_exact(&mut seed_bytes).unwrap();
 
-        let phrase = Mnemonic::from_entropy(&seed_bytes, Language::English,).unwrap().phrase().to_string();
+        Mnemonic::from_entropy(&seed_bytes, Language::English) 
+    }.map_err(|e| format!("Failed to read seed. {:?}", e));
 
-        Ok(phrase)
+    phrase.map(|m| m.phrase().to_string())
     }
 
 
@@ -569,14 +595,18 @@ impl LightClient {
             1_000_000, // 1 MB write buffer
             File::create(self.config.get_wallet_path()).unwrap());
         
-        match self.wallet.write().unwrap().write(&mut file_buffer) {
+            let r = match self.wallet.write().unwrap().write(&mut file_buffer) {
             Ok(_) => Ok(()),
             Err(e) => {
                 let err = format!("ERR: {}", e);
                 error!("{}", err);
                 Err(e.to_string())
             }
-        }
+        };
+
+        file_buffer.flush().map_err(|e| format!("{}", e))?;
+
+        r
     }
 
     pub fn get_server_uri(&self) -> http::Uri {
@@ -776,10 +806,12 @@ impl LightClient {
                 // For each sapling note that is not a change, add a Tx.
                 txns.extend(v.notes.iter()
                     .filter( |nd| !nd.is_change )
-                    .map ( |nd| 
+                    .enumerate()
+                    .map ( |(i, nd)| 
                         object! {
                             "block_height" => v.block,
                             "datetime"     => v.datetime,
+                            "position"     => i,
                             "txid"         => format!("{}", v.txid),
                             "amount"       => nd.note.value as i64,
                             "address"      => LightWallet::note_address(self.config.hrp_sapling_address(), nd),
@@ -1295,13 +1327,13 @@ pub mod tests {
             let seed = lc.do_seed_phrase().unwrap()["seed"].as_str().unwrap().to_string();
             lc.do_save().unwrap();
 
-            assert_eq!(seed, LightClient::attempt_recover_seed(&config).unwrap());
+            assert_eq!(seed, LightClient::attempt_recover_seed(&config, None).unwrap());
 
             // Now encrypt and save the file
-            lc.wallet.write().unwrap().encrypt("password".to_string()).unwrap();
+            let pwd = "password".to_string();
-            lc.do_save().unwrap();
+            lc.wallet.write().unwrap().encrypt(pwd.clone()).unwrap();
 
-            assert!(LightClient::attempt_recover_seed(&config).is_err());
+            assert_eq!(seed, LightClient::attempt_recover_seed(&config, Some(pwd)).unwrap());
         }
     }
 

lib/src/lightclient/checkpoints.rs

@@ -32,6 +32,11 @@ fn get_main_checkpoint(height: u64) ->  Option<(u64, &'static str, &'static str)
 
                    "019081dbde619339e0e85a13df8ff833ea866502d96d9839242eaf4e6f89a9935b000f00000001b542d70d235bfef8c3ea401fc7682c4889abc1d8047346ec374a846e498cf44f013317e59dfbc56000b20131e73d531dd805c481978f86c055b2f863ea7f0b296b01e0bc6b4ae0036aecc0973669061d777fe00f33c19d561cb89f9d8fef5d9d35100001d4ce711d7659419f11c307a77ab79c0fc2f62d1e7b2fc650d6a51704bcf31223017c86ee02304db5f5158a4186cc65e9ceb6acdf88877ea59f5184b04ada15bf0c01d3b830cc0959d5c6616a2eb11231763576719f844da23aafeb3ddcd44dcbb262017cda15fd9d8af2559d2fa920983b832ef410d10811cab7e678ee788d4a74df0a0001ef041d21ca2e599aca269d5a63b35f5ac2abe8e776279fb09ae902778b33746301982a0ad78d7d67d3c7b026adfdb342eceb50557cb4677ec43742028c4602216701b8d79586ce15b0cd9b3683091dea42cdad3fa4dc6d7d7853aaac062aa5717527"
         ),
+
+        (220000,  "00000000dd40d7372e60da03205bfc9bd796cc467737e093a58ab08b688014a4",
+
+                   "01cf87373bc969942c2cc6dc2cb399c4eaae1d234cc785b1decae4847eb7a21c04001001f2cd27d01566c2594f6bd1d141bc68c2c5b6d7b9c27f1183780636255332b16b01b85d3ab8c19e4fa3f717e48354cd97d416b1a916b0e642777f61aa4b75359b39000000015ae551df3653030b6d6aaf858df93741290191de721b502059baf88d7d56906c016deb46b4029552954f58f0271104a1a2d16838ce5cb8e6adef9aed3c29ea271c000001911852d57122315ab96dce00ca98cf3b3dfc423759610f047a9efd8f57172656000183e3743bb17818ab1f0af4b5d4f564eabef0ff4fbf7e21c4a8f02c91d9963a3e0139bc58e226e9585cb428cf367505f04984e6cf5d91514622a3895b8a8079cb2a0122fd7dee5fcd9788ea5c5d2d3c7a61ec8e9bc667daf380d8361b888e52788f09000130195f8f73dc864a32004ff13f5fa2fed32723a150a498f593590ccec493e004"
+        ),
     ];
 
     find_checkpoint(height, checkpoints)

lib/src/lightwallet.rs

@@ -11,6 +11,7 @@ use log::{info, warn, error};
 
 use protobuf::parse_from_bytes;
 
+use libflate::{gzip::{Decoder, Encoder}, finish::AutoFinishUnchecked};
 use secp256k1::SecretKey;
 use bip39::{Mnemonic, Language};
 
@@ -134,7 +135,7 @@ pub struct LightWallet {
 
 impl LightWallet {
     pub fn serialized_version() -> u64 {
-        return 4;
+        return 5;
     }
 
     fn get_taddr_from_bip39seed(config: &LightClientConfig, bip39_seed: &[u8], pos: u32) -> SecretKey {
@@ -232,22 +233,32 @@ impl LightWallet {
         })
     }
 
-    pub fn read<R: Read>(mut reader: R, config: &LightClientConfig) -> io::Result<Self> {
+    pub fn read<R: Read>(mut inp: R, config: &LightClientConfig) -> io::Result<Self> {
-        let version = reader.read_u64::<LittleEndian>()?;
+        let version = inp.read_u64::<LittleEndian>()?;
         if version > LightWallet::serialized_version() {
             let e = format!("Don't know how to read wallet version {}. Do you have the latest version?", version);
             error!("{}", e);
             return Err(io::Error::new(ErrorKind::InvalidData, e));
         }
-
+    println!("Reading wallet version {}", version);
         info!("Reading wallet version {}", version);
 
+         // After version 5, we're writing the rest of the file as a compressed stream (gzip)
+         let mut reader: Box<dyn Read> = if version <= 4 {
+            info!("Reading direct");
+            Box::new(inp)
+        } else {
+            info!("Reading libflat");
+            Box::new(Decoder::new(inp).unwrap())
+        };
+
         let encrypted = if version >= 4 {
             reader.read_u8()? > 0
         } else {
             false
         };
      
+        info!("Wallet Encryption {:?}", encrypted);
         let mut enc_seed = [0u8; 48];
         if version >= 4 {
             reader.read_exact(&mut enc_seed)?;
@@ -331,14 +342,17 @@ impl LightWallet {
         })
     }
 
-    pub fn write<W: Write>(&self, mut writer: W) -> io::Result<()> {
+    pub fn write<W: Write>(&self, mut out: W) -> io::Result<()> {
         if self.encrypted && self.unlocked {
             return Err(Error::new(ErrorKind::InvalidInput, 
                         format!("Cannot write while wallet is unlocked while encrypted.")));
         }
 
         // Write the version
-        writer.write_u64::<LittleEndian>(LightWallet::serialized_version())?;
+        out.write_u64::<LittleEndian>(LightWallet::serialized_version())?;
+
+        // Gzip encoder
+        let mut writer = AutoFinishUnchecked::new(Encoder::new(out).unwrap());
 
         // Write if it is locked
         writer.write_u8(if self.encrypted {1} else {0})?;
@@ -387,9 +401,7 @@ impl LightWallet {
 
         // While writing the birthday, get it from the fn so we recalculate it properly
         // in case of rescans etc...
-        writer.write_u64::<LittleEndian>(self.get_birthday())?;
+        writer.write_u64::<LittleEndian>(self.get_birthday())
-
-        Ok(())
     }
 
     pub fn note_address(hrp: &str, note: &SaplingNoteData) -> Option<String> {
@@ -1088,14 +1100,20 @@ impl LightWallet {
                 };
 
                 {
-                    info!("A sapling note was spent in {}", tx.txid());
+                    info!("A sapling note was sent in {}, getting memo", tx.txid());
-                    // Update the WalletTx 
+
                     // Do it in a short scope because of the write lock.   
                     let mut txs = self.txs.write().unwrap();
-                    txs.get_mut(&tx.txid()).unwrap()
+                       // Update memo if we have this Tx. 
-                        .notes.iter_mut()
+                       match txs.get_mut(&tx.txid())
-                        .find(|nd| nd.note == note).unwrap()
+                       .and_then(|t| {
-                        .memo = Some(memo);
+                           t.notes.iter_mut().find(|nd| nd.note == note)
+                       }) {
+                           None => (),
+                           Some(nd) => {
+                               nd.memo = Some(memo)
+                           }
+                       }
                 }
             }
 
@@ -1137,7 +1155,7 @@ impl LightWallet {
 
                                 let mut txs = self.txs.write().unwrap();
                                 if txs.get(&tx.txid()).unwrap().outgoing_metadata.iter()
-                                        .find(|om| om.address == address && om.value == note.value)
+                                        .find(|om| om.address == address && om.value == note.value  && om.memo == memo)
                                         .is_some() {
                                     warn!("Duplicate outgoing metadata");
                                     continue;
@@ -1453,16 +1471,16 @@ impl LightWallet {
             return Err("Need at least one destination address".to_string());
         }
 
-        // Check for duplicates in the to list
+        // Check for duplicates in the to list - We need that for HushChat
-        if tos.len() > 1 {
+       // if tos.len() > 1 {
-            let mut to_addresses = tos.iter().map(|t| t.0.to_string()).collect::<Vec<_>>();
+       //     let mut to_addresses = tos.iter().map(|t| t.0.to_string()).collect::<Vec<_>>();
-            to_addresses.sort();
+       //     to_addresses.sort();
-            for i in 0..to_addresses.len()-1 {
+       //     for i in 0..to_addresses.len()-1 {
-                if to_addresses[i] == to_addresses[i+1] {
+       //         if to_addresses[i] == to_addresses[i+1] {
-                    return Err(format!("To address {} is duplicated", to_addresses[i]));
+       //             return Err(format!("To address {} is duplicated", to_addresses[i]));
-                }
+       //         }
-            }
+       //     }
-        }
+       // }
 
         let total_value = tos.iter().map(|to| to.1).sum::<u64>() as u64;
         println!(
@@ -1608,7 +1626,17 @@ impl LightWallet {
 
         for (to, value, memo) in recepients {
             // Compute memo if it exists
-            let encoded_memo = memo.map(|s| Memo::from_str(&s).unwrap());
+            let encoded_memo = match memo {
+                None => None,
+                Some(s) => match Memo::from_str(&s) {
+                    None => {
+                        let e = format!("Error creating output. Memo {:?} is too long", s);
+                        error!("{}", e);
+                        return Err(e);
+                    },
+                    Some(m) => Some(m)
+                }
+            };
             
             println!("{}: Adding output", now() - start_time);
 

lib/src/lightwallet/tests.rs

@@ -1518,18 +1518,32 @@ fn test_bad_send() {
                                         vec![(&ext_taddr, AMOUNT1 + 10, None)]);
     assert!(raw_tx.err().unwrap().contains("Insufficient verified funds"));
 
-    // Duplicated addresses
-    let raw_tx = wallet.send_to_address(branch_id, &ss, &so,
-                                        vec![(&ext_taddr, AMOUNT1 + 10, None),
-                                             (&ext_taddr, AMOUNT1 + 10, None)]);
-    assert!(raw_tx.err().unwrap().contains("duplicate"));
-
     // No addresses
     let raw_tx = wallet.send_to_address(branch_id, &ss, &so, vec![]);
     assert!(raw_tx.err().unwrap().contains("at least one"));
 
 }
 
+#[test]
+fn test_duplicate_outputs() {
+    // Test all the ways in which a send should fail
+    const AMOUNT1: u64 = 50000;
+    let _fee: u64 = DEFAULT_FEE.try_into().unwrap();
+
+    let (wallet, _txid1, _block_hash) = get_test_wallet(AMOUNT1);
+
+    let branch_id = u32::from_str_radix("2bb40e60", 16).unwrap();
+    let (ss, so) = get_sapling_params().unwrap();
+    let ext_taddr = wallet.address_from_sk(&SecretKey::from_slice(&[1u8; 32]).unwrap());  
+
+    // Duplicated addresses with memos are fine too
+    let raw_tx = wallet.send_to_address(branch_id, &ss, &so,
+        vec![(&ext_taddr, 100, Some("First memo".to_string())),
+             (&ext_taddr, 0, Some("Second memo".to_string())),
+             (&ext_taddr, 0, Some("Third memo".to_string()))]);
+    assert!(raw_tx.is_ok());
+}
+
 #[test]
 #[should_panic]
 fn test_bad_params() {

mkrelease.sh

@@ -25,6 +25,9 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 
 if [ -z $APP_VERSION ]; then echo "APP_VERSION is not set"; exit 1; fi
 
+# Write the version file
+echo "pub const VERSION:&str = \"$APP_VERSION\";" > /cli/src/version.rs
+
 # First, do the tests
 cd lib && cargo test --release
 retVal=$?