From 243b6a9d4b54485ca4ff056fc35b77767e8ba06e Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Tue, 3 Oct 2017 17:35:32 +0100
Subject: [PATCH 01/11] Add connections in BIP65 and BIP66 tests to the test
 manager

Fixes a bug in the tests causing them to silently pass instead of correctly
reporting other errors. Introduced in 4a785b0a5ba9689e0a76f202c34e0d310a7487d7
during the test rewrites.
---
 qa/rpc-tests/bip65-cltv-p2p.py | 2 +-
 qa/rpc-tests/bipdersig-p2p.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py
index 6f31c7663..8fef45afe 100755
--- a/qa/rpc-tests/bip65-cltv-p2p.py
+++ b/qa/rpc-tests/bip65-cltv-p2p.py
@@ -38,7 +38,7 @@ class BIP65Test(ComparisonTestFramework):
 
     def run_test(self):
         test = TestManager(self, self.options.tmpdir)
-        # Don't call test.add_all_connections because there is only one node.
+        test.add_all_connections(self.nodes)
         NetworkThread().start() # Start up network handling in another thread
         test.run()
 
diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py
index 9604cdbdd..523a694bf 100755
--- a/qa/rpc-tests/bipdersig-p2p.py
+++ b/qa/rpc-tests/bipdersig-p2p.py
@@ -37,7 +37,7 @@ class BIP66Test(ComparisonTestFramework):
 
     def run_test(self):
         test = TestManager(self, self.options.tmpdir)
-        # Don't call test.add_all_connections because there is only one node.
+        test.add_all_connections(self.nodes)
         NetworkThread().start() # Start up network handling in another thread
         test.run()
 

From 13ca1e8011c8794d2b05861d3b0c9dc284965c1b Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 13 Jul 2017 14:23:33 -0500
Subject: [PATCH 02/11] [Test] MiniNode: Implement JSDescription parsing

---
 qa/rpc-tests/test_framework/mininode.py | 148 ++++++++++++++++++++++++
 1 file changed, 148 insertions(+)

diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index c39637073..7006e68e4 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -307,6 +307,154 @@ class CBlockLocator(object):
             % (self.nVersion, repr(self.vHave))
 
 
+G1_PREFIX_MASK = 0x02
+G2_PREFIX_MASK = 0x0a
+
+class ZCProof(object):
+    def __init__(self):
+        self.g_A = None
+        self.g_A_prime = None
+        self.g_B = None
+        self.g_B_prime = None
+        self.g_C = None
+        self.g_C_prime = None
+        self.g_K = None
+        self.g_H = None
+
+    def deserialize(self, f):
+        def deser_g1(self, f):
+            leadingByte = struct.unpack("<B", f.read(1))[0]
+            return {
+                'y_lsb': leadingByte & 1,
+                'x': f.read(32),
+            }
+        def deser_g2(self, f):
+            leadingByte = struct.unpack("<B", f.read(1))[0]
+            return {
+                'y_gt': leadingByte & 1,
+                'x': f.read(64),
+            }
+        self.g_A = deser_g1(f)
+        self.g_A_prime = deser_g1(f)
+        self.g_B = deser_g2(f)
+        self.g_B_prime = deser_g1(f)
+        self.g_C = deser_g1(f)
+        self.g_C_prime = deser_g1(f)
+        self.g_K = deser_g1(f)
+        self.g_H = deser_g1(f)
+
+    def serialize(self):
+        def ser_g1(self, p):
+            return chr(G1_PREFIX_MASK | p['y_lsb']) + p['x']
+        def ser_g2(self, p):
+            return chr(G2_PREFIX_MASK | p['y_gt']) + p['x']
+        r = ""
+        r += ser_g1(self.g_A)
+        r += ser_g1(self.g_A_prime)
+        r += ser_g2(self.g_B)
+        r += ser_g1(self.g_B_prime)
+        r += ser_g1(self.g_C)
+        r += ser_g1(self.g_C_prime)
+        r += ser_g1(self.g_K)
+        r += ser_g1(self.g_H)
+        return r
+
+    def __repr__(self):
+        return "ZCProof(g_A=%s g_A_prime=%s g_B=%s g_B_prime=%s g_C=%s g_C_prime=%s g_K=%s g_H=%s)" \
+            % (repr(self.g_A), repr(self.g_A_prime),
+               repr(self.g_B), repr(self.g_B_prime),
+               repr(self.g_C), repr(self.g_C_prime),
+               repr(self.g_K), repr(self.g_H))
+
+
+ZC_NUM_JS_INPUTS = 2
+ZC_NUM_JS_OUTPUTS = 2
+
+ZC_NOTEPLAINTEXT_LEADING = 1
+ZC_V_SIZE = 8
+ZC_RHO_SIZE = 32
+ZC_R_SIZE = 32
+ZC_MEMO_SIZE = 512
+
+ZC_NOTEPLAINTEXT_SIZE = (
+  ZC_NOTEPLAINTEXT_LEADING +
+  ZC_V_SIZE +
+  ZC_RHO_SIZE +
+  ZC_R_SIZE +
+  ZC_MEMO_SIZE
+)
+
+NOTEENCRYPTION_AUTH_BYTES = 16
+
+ZC_NOTECIPHERTEXT_SIZE = (
+  ZC_NOTEPLAINTEXT_SIZE +
+  NOTEENCRYPTION_AUTH_BYTES
+)
+
+class JSDescription(object):
+    def __init__(self):
+        self.vpub_old = 0
+        self.vpub_new = 0
+        self.anchor = 0
+        self.nullifiers = [0] * ZC_NUM_JS_INPUTS
+        self.commitments = [0] * ZC_NUM_JS_OUTPUTS
+        self.onetimePubKey = 0
+        self.randomSeed = 0
+        self.macs = [0] * ZC_NUM_JS_INPUTS
+        self.proof = None
+        self.ciphertexts = [None] * ZC_NUM_JS_OUTPUTS
+
+    def deserialize(self, f):
+        self.vpub_old = struct.unpack("<q", f.read(8))[0]
+        self.vpub_new = struct.unpack("<q", f.read(8))[0]
+        self.anchor = deser_uint256(f)
+
+        self.nullifiers = []
+        for i in range(ZC_NUM_JS_INPUTS):
+            self.nullifiers.append(deser_uint256(f))
+
+        self.commitments = []
+        for i in range(ZC_NUM_JS_OUTPUTS):
+            self.commitments.append(deser_uint256(f))
+
+        self.onetimePubKey = deser_uint256(f)
+        self.randomSeed = deser_uint256(f)
+
+        self.macs = []
+        for i in range(ZC_NUM_JS_INPUTS):
+            self.macs.append(deser_uint256(f))
+
+        self.proof = ZCProof()
+        self.proof.deserialize(f)
+
+        self.ciphertexts = []
+        for i in range(ZC_NUM_JS_OUTPUTS):
+            self.ciphertexts.append(f.read(ZC_NOTECIPHERTEXT_SIZE))
+
+    def serialize(self):
+        r = ""
+        r += struct.pack("<q", self.vpub_old)
+        r += struct.pack("<q", self.vpub_new)
+        r += ser_uint256(self.anchor)
+        for i in range(ZC_NUM_JS_INPUTS):
+            r += ser_uint256(self.nullifiers[i])
+        for i in range(ZC_NUM_JS_OUTPUTS):
+            r += ser_uint256(self.commitments[i])
+        r += ser_uint256(self.onetimePubKey)
+        r += ser_uint256(self.randomSeed)
+        for i in range(ZC_NUM_JS_INPUTS):
+            r += ser_uint256(self.macs[i])
+        r += self.proof.serialize()
+        for i in range(ZC_NUM_JS_OUTPUTS):
+            r += ser_uint256(self.ciphertexts[i])
+        return r
+
+    def __repr__(self):
+        return "JSDescription(vpub_old=%i.%08i vpub_new=%i.%08i anchor=%064x onetimePubKey=%064x randomSeed=%064x proof=%s)" \
+            % (self.vpub_old, self.vpub_new, self.anchor,
+               self.onetimePubKey, self.randomSeed, repr(self.proof))
+
+
 class COutPoint(object):
     def __init__(self, hash=0, n=0):
         self.hash = hash

From c47dc872a7b0461c36332ffb6fed53e72fe45080 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 13 Jul 2017 14:24:16 -0500
Subject: [PATCH 03/11] [Test] MiniNode: Implement v2 CTransaction parsing

---
 qa/rpc-tests/test_framework/mininode.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index 7006e68e4..7ebf685e2 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -530,6 +530,9 @@ class CTransaction(object):
             self.vin = []
             self.vout = []
             self.nLockTime = 0
+            self.vjoinsplit = []
+            self.joinSplitPubKey = None
+            self.joinSplitSig = None
             self.sha256 = None
             self.hash = None
         else:
@@ -537,6 +540,9 @@ class CTransaction(object):
             self.vin = copy.deepcopy(tx.vin)
             self.vout = copy.deepcopy(tx.vout)
             self.nLockTime = tx.nLockTime
+            self.vjoinsplit = copy.deepcopy(tx.vjoinsplit)
+            self.joinSplitPubKey = tx.joinSplitPubKey
+            self.joinSplitSig = tx.joinSplitSig
             self.sha256 = None
             self.hash = None
 
@@ -545,6 +551,11 @@ class CTransaction(object):
         self.vin = deser_vector(f, CTxIn)
         self.vout = deser_vector(f, CTxOut)
         self.nLockTime = struct.unpack("<I", f.read(4))[0]
+        if self.nVersion >= 2:
+            self.vjoinsplit = deser_vector(f, JSDescription)
+            if len(self.vjoinsplit) > 0:
+                self.joinSplitPubKey = deser_uint256(f)
+                self.joinSplitSig = f.read(64)
         self.sha256 = None
         self.hash = None
 
@@ -554,6 +565,11 @@ class CTransaction(object):
         r += ser_vector(self.vin)
         r += ser_vector(self.vout)
         r += struct.pack("<I", self.nLockTime)
+        if self.nVersion >= 2:
+            r += ser_vector(self.vjoinsplit)
+            if len(self.vjoinsplit) > 0:
+                r += ser_uint256(self.joinSplitPubKey)
+                r += self.joinSplitSig
         return r
 
     def rehash(self):
@@ -573,8 +589,15 @@ class CTransaction(object):
         return True
 
     def __repr__(self):
-        return "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i)" \
+        r = "CTransaction(nVersion=%i vin=%s vout=%s nLockTime=%i" \
             % (self.nVersion, repr(self.vin), repr(self.vout), self.nLockTime)
+        if self.nVersion >= 2:
+            r += " vjoinsplit=%s" % repr(self.vjoinsplit)
+            if len(self.vjoinsplit) > 0:
+                r += " joinSplitPubKey=%064x joinSplitSig=%064x" \
+                    (self.joinSplitPubKey, self.joinSplitSig)
+        r += ")"
+        return r
 
 
 class CBlockHeader(object):

From 31bc3d25818c60f0cb3b6e8a8ecdfa28d367fd58 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 13 Jul 2017 14:25:10 -0500
Subject: [PATCH 04/11] [Test] MiniNode: Implement Zcash block parsing

---
 qa/rpc-tests/test_framework/mininode.py | 59 +++++++++++++++++++++----
 1 file changed, 50 insertions(+), 9 deletions(-)

diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index 7ebf685e2..e35503bfb 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -234,6 +234,36 @@ def ser_int_vector(l):
     return r
 
 
+def deser_char_vector(f):
+    nit = struct.unpack("<B", f.read(1))[0]
+    if nit == 253:
+        nit = struct.unpack("<H", f.read(2))[0]
+    elif nit == 254:
+        nit = struct.unpack("<I", f.read(4))[0]
+    elif nit == 255:
+        nit = struct.unpack("<Q", f.read(8))[0]
+    r = []
+    for i in xrange(nit):
+        t = struct.unpack("<B", f.read(1))[0]
+        r.append(t)
+    return r
+
+
+def ser_char_vector(l):
+    r = ""
+    if len(l) < 253:
+        r = chr(len(l))
+    elif len(l) < 0x10000:
+        r = chr(253) + struct.pack("<H", len(l))
+    elif len(l) < 0x100000000L:
+        r = chr(254) + struct.pack("<I", len(l))
+    else:
+        r = chr(255) + struct.pack("<Q", len(l))
+    for i in l:
+        r += chr(i)
+    return r
+
+
 # Objects that map to bitcoind objects, which can be serialized/deserialized
 
 class CAddress(object):
@@ -608,20 +638,24 @@ class CBlockHeader(object):
             self.nVersion = header.nVersion
             self.hashPrevBlock = header.hashPrevBlock
             self.hashMerkleRoot = header.hashMerkleRoot
+            self.hashReserved = header.hashReserved
             self.nTime = header.nTime
             self.nBits = header.nBits
             self.nNonce = header.nNonce
+            self.nSolution = header.nSolution
             self.sha256 = header.sha256
             self.hash = header.hash
             self.calc_sha256()
 
     def set_null(self):
-        self.nVersion = 1
+        self.nVersion = 4
         self.hashPrevBlock = 0
         self.hashMerkleRoot = 0
+        self.hashReserved = 0
         self.nTime = 0
         self.nBits = 0
         self.nNonce = 0
+        self.nSolution = []
         self.sha256 = None
         self.hash = None
 
@@ -629,9 +663,11 @@ class CBlockHeader(object):
         self.nVersion = struct.unpack("<i", f.read(4))[0]
         self.hashPrevBlock = deser_uint256(f)
         self.hashMerkleRoot = deser_uint256(f)
+        self.hashReserved = deser_uint256(f)
         self.nTime = struct.unpack("<I", f.read(4))[0]
         self.nBits = struct.unpack("<I", f.read(4))[0]
-        self.nNonce = struct.unpack("<I", f.read(4))[0]
+        self.nNonce = deser_uint256(f)
+        self.nSolution = deser_char_vector(f)
         self.sha256 = None
         self.hash = None
 
@@ -640,9 +676,11 @@ class CBlockHeader(object):
         r += struct.pack("<i", self.nVersion)
         r += ser_uint256(self.hashPrevBlock)
         r += ser_uint256(self.hashMerkleRoot)
+        r += ser_uint256(self.hashReserved)
         r += struct.pack("<I", self.nTime)
         r += struct.pack("<I", self.nBits)
-        r += struct.pack("<I", self.nNonce)
+        r += ser_uint256(self.nNonce)
+        r += ser_char_vector(self.nSolution)
         return r
 
     def calc_sha256(self):
@@ -651,9 +689,11 @@ class CBlockHeader(object):
             r += struct.pack("<i", self.nVersion)
             r += ser_uint256(self.hashPrevBlock)
             r += ser_uint256(self.hashMerkleRoot)
+            r += ser_uint256(self.hashReserved)
             r += struct.pack("<I", self.nTime)
             r += struct.pack("<I", self.nBits)
-            r += struct.pack("<I", self.nNonce)
+            r += ser_uint256(self.nNonce)
+            r += ser_char_vector(self.nSolution)
             self.sha256 = uint256_from_str(hash256(r))
             self.hash = hash256(r)[::-1].encode('hex_codec')
 
@@ -663,9 +703,9 @@ class CBlockHeader(object):
         return self.sha256
 
     def __repr__(self):
-        return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \
-            % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
-               time.ctime(self.nTime), self.nBits, self.nNonce)
+        return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x hashReserved=%064x nTime=%s nBits=%08x nNonce=%064x nSolution=%s)" \
+            % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, self.hashReserved,
+               time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.nSolution))
 
 
 class CBlock(CBlockHeader):
@@ -716,9 +756,10 @@ class CBlock(CBlockHeader):
             self.rehash()
 
     def __repr__(self):
-        return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \
+        return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x hashReserved=%064x nTime=%s nBits=%08x nNonce=%064x nSolution=%s vtx=%s)" \
             % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
-               time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
+               self.hashReserved, time.ctime(self.nTime), self.nBits,
+               self.nNonce, repr(self.nSolution), repr(self.vtx))
 
 
 class CUnsignedAlert(object):

From 0de9a481abc83868f3c9bb0ba2da4e76048ea046 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 13 Jul 2017 14:25:31 -0500
Subject: [PATCH 05/11] [Test] MiniNode: Update protocol version and network
 magics

---
 qa/rpc-tests/test_framework/mininode.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index e35503bfb..c418d3f63 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -32,7 +32,7 @@ import logging
 import copy
 
 BIP0031_VERSION = 60000
-MY_VERSION = 60001  # past bip-31 for ping/pong
+MY_VERSION = 170002  # past bip-31 for ping/pong
 MY_SUBVERSION = "/python-mininode-tester:0.0.1/"
 
 MAX_INV_SZ = 50000
@@ -1294,9 +1294,9 @@ class NodeConn(asyncore.dispatcher):
         "mempool": msg_mempool
     }
     MAGIC_BYTES = {
-        "mainnet": "\xf9\xbe\xb4\xd9",   # mainnet
-        "testnet3": "\x0b\x11\x09\x07",  # testnet3
-        "regtest": "\xfa\xbf\xb5\xda"    # regtest
+        "mainnet": "\x24\xe9\x27\x64",   # mainnet
+        "testnet3": "\xfa\x1a\xf9\xbf",  # testnet3
+        "regtest": "\xaa\xe8\x3f\x5f"    # regtest
     }
 
     def __init__(self, dstaddr, dstport, rpc, callback, net="regtest"):

From e68c3ec1887d340927ba573ef8dc625d1a4a7d9e Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 13 Jul 2017 17:04:53 -0500
Subject: [PATCH 06/11] [Test] MiniNode: Use Zcash PoW

Equihash solver code extracted from https://github.com/str4d/zcash-pow

RPC tests now require pyblake2 to be installed
---
 qa/rpc-tests/test_framework/blocktools.py |   7 +-
 qa/rpc-tests/test_framework/equihash.py   | 293 ++++++++++++++++++++++
 qa/rpc-tests/test_framework/mininode.py   |  37 ++-
 3 files changed, 330 insertions(+), 7 deletions(-)
 create mode 100755 qa/rpc-tests/test_framework/equihash.py

diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py
index 0f8147253..cffa109cf 100644
--- a/qa/rpc-tests/test_framework/blocktools.py
+++ b/qa/rpc-tests/test_framework/blocktools.py
@@ -7,7 +7,7 @@
 from mininode import CBlock, CTransaction, CTxIn, CTxOut, COutPoint, ser_string
 
 # Create a block (with regtest difficulty)
-def create_block(hashprev, coinbase, nTime=None):
+def create_block(hashprev, coinbase, nTime=None, nBits=None):
     block = CBlock()
     if nTime is None:
         import time
@@ -15,7 +15,10 @@ def create_block(hashprev, coinbase, nTime=None):
     else:
         block.nTime = nTime
     block.hashPrevBlock = hashprev
-    block.nBits = 0x207fffff # Will break after a difficulty adjustment...
+    if nBits is None:
+        block.nBits = 0x200f0f0f # Will break after a difficulty adjustment...
+    else:
+        block.nBits = nBits
     block.vtx.append(coinbase)
     block.hashMerkleRoot = block.calc_merkle_root()
     block.calc_sha256()
diff --git a/qa/rpc-tests/test_framework/equihash.py b/qa/rpc-tests/test_framework/equihash.py
new file mode 100755
index 000000000..e40451978
--- /dev/null
+++ b/qa/rpc-tests/test_framework/equihash.py
@@ -0,0 +1,293 @@
+from operator import itemgetter
+import struct
+
+DEBUG = False
+VERBOSE = False
+
+
+word_size = 32
+word_mask = (1<<word_size)-1
+
+def expand_array(inp, out_len, bit_len, byte_pad=0):
+    assert bit_len >= 8 and word_size >= 7+bit_len
+    bit_len_mask = (1<<bit_len)-1
+
+    out_width = (bit_len+7)/8 + byte_pad
+    assert out_len == 8*out_width*len(inp)/bit_len
+    out = bytearray(out_len)
+
+    bit_len_mask = (1 << bit_len) - 1
+
+    # The acc_bits least-significant bits of acc_value represent a bit sequence
+    # in big-endian order.
+    acc_bits = 0;
+    acc_value = 0;
+
+    j = 0
+    for i in xrange(len(inp)):
+        acc_value = ((acc_value << 8) & word_mask) | inp[i]
+        acc_bits += 8
+
+        # When we have bit_len or more bits in the accumulator, write the next
+        # output element.
+        if acc_bits >= bit_len:
+            acc_bits -= bit_len
+            for x in xrange(byte_pad, out_width):
+                out[j+x] = (
+                    # Big-endian
+                    acc_value >> (acc_bits+(8*(out_width-x-1)))
+                ) & (
+                    # Apply bit_len_mask across byte boundaries
+                    (bit_len_mask >> (8*(out_width-x-1))) & 0xFF
+                )
+            j += out_width
+
+    return out
+
+def compress_array(inp, out_len, bit_len, byte_pad=0):
+    assert bit_len >= 8 and word_size >= 7+bit_len
+
+    in_width = (bit_len+7)/8 + byte_pad
+    assert out_len == bit_len*len(inp)/(8*in_width)
+    out = bytearray(out_len)
+
+    bit_len_mask = (1 << bit_len) - 1
+
+    # The acc_bits least-significant bits of acc_value represent a bit sequence
+    # in big-endian order.
+    acc_bits = 0;
+    acc_value = 0;
+
+    j = 0
+    for i in xrange(out_len):
+        # When we have fewer than 8 bits left in the accumulator, read the next
+        # input element.
+        if acc_bits < 8:
+            acc_value = ((acc_value << bit_len) & word_mask) | inp[j]
+            for x in xrange(byte_pad, in_width):
+                acc_value = acc_value | (
+                    (
+                        # Apply bit_len_mask across byte boundaries
+                        inp[j+x] & ((bit_len_mask >> (8*(in_width-x-1))) & 0xFF)
+                    ) << (8*(in_width-x-1))); # Big-endian
+            j += in_width
+            acc_bits += bit_len
+
+        acc_bits -= 8
+        out[i] = (acc_value >> acc_bits) & 0xFF
+
+    return out
+
+def get_indices_from_minimal(minimal, bit_len):
+    eh_index_size = 4
+    assert (bit_len+7)/8 <= eh_index_size
+    len_indices = 8*eh_index_size*len(minimal)/bit_len
+    byte_pad = eh_index_size - (bit_len+7)/8
+    expanded = expand_array(minimal, len_indices, bit_len, byte_pad)
+    return [struct.unpack('>I', expanded[i:i+4])[0] for i in range(0, len_indices, eh_index_size)]
+
+def get_minimal_from_indices(indices, bit_len):
+    eh_index_size = 4
+    assert (bit_len+7)/8 <= eh_index_size
+    len_indices = len(indices)*eh_index_size
+    min_len = bit_len*len_indices/(8*eh_index_size)
+    byte_pad = eh_index_size - (bit_len+7)/8
+    byte_indices = bytearray(''.join([struct.pack('>I', i) for i in indices]))
+    return compress_array(byte_indices, min_len, bit_len, byte_pad)
+
+
+def hash_nonce(digest, nonce):
+    for i in range(8):
+        digest.update(struct.pack('<I', nonce >> (32*i)))
+
+def hash_xi(digest, xi):
+    digest.update(struct.pack('<I', xi))
+    return digest # For chaining
+
+def count_zeroes(h):
+    # Convert to binary string
+    if type(h) == bytearray:
+        h = ''.join('{0:08b}'.format(x, 'b') for x in h)
+    else:
+        h = ''.join('{0:08b}'.format(ord(x), 'b') for x in h)
+    # Count leading zeroes
+    return (h+'1').index('1')
+
+def has_collision(ha, hb, i, l):
+    res = [ha[j] == hb[j] for j in range((i-1)*l/8, i*l/8)]
+    return reduce(lambda x, y: x and y, res)
+
+def distinct_indices(a, b):
+    for i in a:
+        for j in b:
+            if i == j:
+                return False
+    return True
+
+def xor(ha, hb):
+    return bytearray(a^b for a,b in zip(ha,hb))
+
+def gbp_basic(digest, n, k):
+    '''Implementation of Basic Wagner's algorithm for the GBP.'''
+    validate_params(n, k)
+    collision_length = n/(k+1)
+    hash_length = (k+1)*((collision_length+7)//8)
+    indices_per_hash_output = 512/n
+
+    # 1) Generate first list
+    if DEBUG: print 'Generating first list'
+    X = []
+    tmp_hash = ''
+    for i in range(0, 2**(collision_length+1)):
+        r = i % indices_per_hash_output
+        if r == 0:
+            # X_i = H(I||V||x_i)
+            curr_digest = digest.copy()
+            hash_xi(curr_digest, i/indices_per_hash_output)
+            tmp_hash = curr_digest.digest()
+        X.append((
+            expand_array(bytearray(tmp_hash[r*n/8:(r+1)*n/8]),
+                         hash_length, collision_length),
+            (i,)
+        ))
+
+    # 3) Repeat step 2 until 2n/(k+1) bits remain
+    for i in range(1, k):
+        if DEBUG: print 'Round %d:' % i
+
+        # 2a) Sort the list
+        if DEBUG: print '- Sorting list'
+        X.sort(key=itemgetter(0))
+        if DEBUG and VERBOSE:
+            for Xi in X[-32:]:
+                print '%s %s' % (print_hash(Xi[0]), Xi[1])
+
+        if DEBUG: print '- Finding collisions'
+        Xc = []
+        while len(X) > 0:
+            # 2b) Find next set of unordered pairs with collisions on first n/(k+1) bits
+            j = 1
+            while j < len(X):
+                if not has_collision(X[-1][0], X[-1-j][0], i, collision_length):
+                    break
+                j += 1
+
+            # 2c) Store tuples (X_i ^ X_j, (i, j)) on the table
+            for l in range(0, j-1):
+                for m in range(l+1, j):
+                    # Check that there are no duplicate indices in tuples i and j
+                    if distinct_indices(X[-1-l][1], X[-1-m][1]):
+                        if X[-1-l][1][0] < X[-1-m][1][0]:
+                            concat = X[-1-l][1] + X[-1-m][1]
+                        else:
+                            concat = X[-1-m][1] + X[-1-l][1]
+                        Xc.append((xor(X[-1-l][0], X[-1-m][0]), concat))
+
+            # 2d) Drop this set
+            while j > 0:
+                X.pop(-1)
+                j -= 1
+        # 2e) Replace previous list with new list
+        X = Xc
+
+    # k+1) Find a collision on last 2n(k+1) bits
+    if DEBUG:
+        print 'Final round:'
+        print '- Sorting list'
+    X.sort(key=itemgetter(0))
+    if DEBUG and VERBOSE:
+        for Xi in X[-32:]:
+            print '%s %s' % (print_hash(Xi[0]), Xi[1])
+    if DEBUG: print '- Finding collisions'
+    solns = []
+    while len(X) > 0:
+        j = 1
+        while j < len(X):
+            if not (has_collision(X[-1][0], X[-1-j][0], k, collision_length) and
+                    has_collision(X[-1][0], X[-1-j][0], k+1, collision_length)):
+                break
+            j += 1
+
+        for l in range(0, j-1):
+            for m in range(l+1, j):
+                res = xor(X[-1-l][0], X[-1-m][0])
+                if count_zeroes(res) == 8*hash_length and distinct_indices(X[-1-l][1], X[-1-m][1]):
+                    if DEBUG and VERBOSE:
+                        print 'Found solution:'
+                        print '- %s %s' % (print_hash(X[-1-l][0]), X[-1-l][1])
+                        print '- %s %s' % (print_hash(X[-1-m][0]), X[-1-m][1])
+                    if X[-1-l][1][0] < X[-1-m][1][0]:
+                        solns.append(list(X[-1-l][1] + X[-1-m][1]))
+                    else:
+                        solns.append(list(X[-1-m][1] + X[-1-l][1]))
+
+        # 2d) Drop this set
+        while j > 0:
+            X.pop(-1)
+            j -= 1
+    return [get_minimal_from_indices(soln, collision_length+1) for soln in solns]
+
+def gbp_validate(digest, minimal, n, k):
+    validate_params(n, k)
+    collision_length = n/(k+1)
+    hash_length = (k+1)*((collision_length+7)//8)
+    indices_per_hash_output = 512/n
+    solution_width = (1 << k)*(collision_length+1)//8
+
+    if len(minimal) != solution_width:
+        print 'Invalid solution length: %d (expected %d)' % \
+            (len(minimal), solution_width)
+        return False
+
+    X = []
+    for i in get_indices_from_minimal(minimal, collision_length+1):
+        r = i % indices_per_hash_output
+        # X_i = H(I||V||x_i)
+        curr_digest = digest.copy()
+        hash_xi(curr_digest, i/indices_per_hash_output)
+        tmp_hash = curr_digest.digest()
+        X.append((
+            expand_array(bytearray(tmp_hash[r*n/8:(r+1)*n/8]),
+                         hash_length, collision_length),
+            (i,)
+        ))
+
+    for r in range(1, k+1):
+        Xc = []
+        for i in range(0, len(X), 2):
+            if not has_collision(X[i][0], X[i+1][0], r, collision_length):
+                print 'Invalid solution: invalid collision length between StepRows'
+                return False
+            if X[i+1][1][0] < X[i][1][0]:
+                print 'Invalid solution: Index tree incorrectly ordered'
+                return False
+            if not distinct_indices(X[i][1], X[i+1][1]):
+                print 'Invalid solution: duplicate indices'
+                return False
+            Xc.append((xor(X[i][0], X[i+1][0]), X[i][1] + X[i+1][1]))
+        X = Xc
+
+    if len(X) != 1:
+        print 'Invalid solution: incorrect length after end of rounds: %d' % len(X)
+        return False
+
+    if count_zeroes(X[0][0]) != 8*hash_length:
+        print 'Invalid solution: incorrect number of zeroes: %d' % count_zeroes(X[0][0])
+        return False
+
+    return True
+
+def zcash_person(n, k):
+    return b'ZcashPoW' + struct.pack('<II', n, k)
+
+def print_hash(h):
+    if type(h) == bytearray:
+        return ''.join('{0:02x}'.format(x, 'x') for x in h)
+    else:
+        return ''.join('{0:02x}'.format(ord(x), 'x') for x in h)
+
+def validate_params(n, k):
+    if (k >= n):
+        raise ValueError('n must be larger than k')
+    if (((n/(k+1))+1) >= 32):
+        raise ValueError('Parameters must satisfy n/(k+1)+1 < 32')
diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py
index c418d3f63..c74d3f931 100755
--- a/qa/rpc-tests/test_framework/mininode.py
+++ b/qa/rpc-tests/test_framework/mininode.py
@@ -30,6 +30,14 @@ from threading import RLock
 from threading import Thread
 import logging
 import copy
+from pyblake2 import blake2b
+
+from .equihash import (
+    gbp_basic,
+    gbp_validate,
+    hash_nonce,
+    zcash_person,
+)
 
 BIP0031_VERSION = 60000
 MY_VERSION = 170002  # past bip-31 for ping/pong
@@ -736,7 +744,13 @@ class CBlock(CBlockHeader):
             hashes = newhashes
         return uint256_from_str(hashes[0])
 
-    def is_valid(self):
+    def is_valid(self, n=48, k=5):
+        # H(I||...
+        digest = blake2b(digest_size=(512/n)*n/8, person=zcash_person(n, k))
+        digest.update(super(CBlock, self).serialize()[:108])
+        hash_nonce(digest, self.nNonce)
+        if not gbp_validate(self.nSolution, digest, n, k):
+            return False
         self.calc_sha256()
         target = uint256_from_compact(self.nBits)
         if self.sha256 > target:
@@ -748,12 +762,25 @@ class CBlock(CBlockHeader):
             return False
         return True
 
-    def solve(self):
-        self.calc_sha256()
+    def solve(self, n=48, k=5):
         target = uint256_from_compact(self.nBits)
-        while self.sha256 > target:
+        # H(I||...
+        digest = blake2b(digest_size=(512/n)*n/8, person=zcash_person(n, k))
+        digest.update(super(CBlock, self).serialize()[:108])
+        self.nNonce = 0
+        while True:
+            # H(I||V||...
+            curr_digest = digest.copy()
+            hash_nonce(curr_digest, self.nNonce)
+            # (x_1, x_2, ...) = A(I, V, n, k)
+            solns = gbp_basic(curr_digest, n, k)
+            for soln in solns:
+                assert(gbp_validate(curr_digest, soln, n, k))
+                self.nSolution = soln
+                self.rehash()
+                if self.sha256 <= target:
+                    return
             self.nNonce += 1
-            self.rehash()
 
     def __repr__(self):
         return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x hashReserved=%064x nTime=%s nBits=%08x nNonce=%064x nSolution=%s vtx=%s)" \

From 643235859afb01fdc874f8e09ca1855b55fb8a97 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Wed, 4 Oct 2017 00:25:06 +0100
Subject: [PATCH 07/11] [Test] MiniNode: Fix coinbase creation

CScriptNum is only used for heights > 16.
---
 qa/rpc-tests/test_framework/blocktools.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py
index cffa109cf..ca566e4bc 100644
--- a/qa/rpc-tests/test_framework/blocktools.py
+++ b/qa/rpc-tests/test_framework/blocktools.py
@@ -4,7 +4,8 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
 
-from mininode import CBlock, CTransaction, CTxIn, CTxOut, COutPoint, ser_string
+from mininode import CBlock, CTransaction, CTxIn, CTxOut, COutPoint
+from script import CScript, OP_0
 
 # Create a block (with regtest difficulty)
 def create_block(hashprev, coinbase, nTime=None, nBits=None):
@@ -45,7 +46,7 @@ def create_coinbase(heightAdjust = 0):
     global counter
     coinbase = CTransaction()
     coinbase.vin.append(CTxIn(COutPoint(0, 0xffffffff), 
-                ser_string(serialize_script_num(counter+heightAdjust)), 0xffffffff))
+                CScript([counter+heightAdjust, OP_0]), 0xffffffff))
     counter += 1
     coinbaseoutput = CTxOut()
     coinbaseoutput.nValue = 50*100000000

From 7596a4922d8e1b43cdc03ddaabcad538375f3ab3 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Wed, 4 Oct 2017 00:28:40 +0100
Subject: [PATCH 08/11] [Test] MiniNode: Coerce OP_PUSHDATA bytearrays to bytes

If a bytearray is passed in as part of an iterable, the CScript constructor
fails because b''.join() cannot be used to join a bytearray to a bytes or str in
Python 2.
---
 qa/rpc-tests/test_framework/script.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/rpc-tests/test_framework/script.py b/qa/rpc-tests/test_framework/script.py
index ab7f4ec74..55a7f8e51 100644
--- a/qa/rpc-tests/test_framework/script.py
+++ b/qa/rpc-tests/test_framework/script.py
@@ -666,7 +666,7 @@ class CScript(bytes):
             else:
                 other = CScriptOp.encode_op_pushdata(bignum.bn2vch(other))
         elif isinstance(other, (bytes, bytearray)):
-            other = CScriptOp.encode_op_pushdata(other)
+            other = bytes(CScriptOp.encode_op_pushdata(other))
         return other
 
     def __add__(self, other):

From c10c40779d9b73359ee61f85ec5f1bb88acac61d Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Wed, 4 Oct 2017 00:34:32 +0100
Subject: [PATCH 09/11] [Test] MiniNode: Implement Zcash coinbase

---
 qa/rpc-tests/test_framework/blocktools.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/qa/rpc-tests/test_framework/blocktools.py b/qa/rpc-tests/test_framework/blocktools.py
index ca566e4bc..1fe2a5dda 100644
--- a/qa/rpc-tests/test_framework/blocktools.py
+++ b/qa/rpc-tests/test_framework/blocktools.py
@@ -5,7 +5,7 @@
 #
 
 from mininode import CBlock, CTransaction, CTxIn, CTxOut, COutPoint
-from script import CScript, OP_0
+from script import CScript, OP_0, OP_EQUAL, OP_HASH160
 
 # Create a block (with regtest difficulty)
 def create_block(hashprev, coinbase, nTime=None, nBits=None):
@@ -49,11 +49,21 @@ def create_coinbase(heightAdjust = 0):
                 CScript([counter+heightAdjust, OP_0]), 0xffffffff))
     counter += 1
     coinbaseoutput = CTxOut()
-    coinbaseoutput.nValue = 50*100000000
+    coinbaseoutput.nValue = int(12.5*100000000)
     halvings = int((counter+heightAdjust)/150) # regtest
     coinbaseoutput.nValue >>= halvings
     coinbaseoutput.scriptPubKey = ""
     coinbase.vout = [ coinbaseoutput ]
+    if halvings == 0: # regtest
+        froutput = CTxOut()
+        froutput.nValue = coinbaseoutput.nValue / 5
+        # regtest
+        fraddr = bytearray([0x67, 0x08, 0xe6, 0x67, 0x0d, 0xb0, 0xb9, 0x50,
+                            0xda, 0xc6, 0x80, 0x31, 0x02, 0x5c, 0xc5, 0xb6,
+                            0x32, 0x13, 0xa4, 0x91])
+        froutput.scriptPubKey = CScript([OP_HASH160, fraddr, OP_EQUAL])
+        coinbaseoutput.nValue -= froutput.nValue
+        coinbase.vout = [ coinbaseoutput, froutput ]
     coinbase.calc_sha256()
     return coinbase
 

From 5455ca0d0e9b39d6844133edfc01107031f2c594 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Wed, 4 Oct 2017 01:21:09 +0100
Subject: [PATCH 10/11] Fix BIP65 and BIP66 tests

Blocks were being created that didn't satisfy the regtest consensus rules.
---
 qa/rpc-tests/bip65-cltv-p2p.py | 10 +++++++---
 qa/rpc-tests/bipdersig-p2p.py  | 11 +++++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py
index 8fef45afe..cfd2df01e 100755
--- a/qa/rpc-tests/bip65-cltv-p2p.py
+++ b/qa/rpc-tests/bip65-cltv-p2p.py
@@ -12,7 +12,6 @@ from test_framework.comptool import TestInstance, TestManager
 from test_framework.script import CScript, OP_1NEGATE, OP_NOP2, OP_DROP
 from binascii import unhexlify
 import cStringIO
-import time
 
 
 '''
@@ -64,9 +63,9 @@ class BIP65Test(ComparisonTestFramework):
 
     def get_tests(self):
         self.coinbase_blocks = self.nodes[0].generate(1)
+        self.nodes[0].generate(100)
         self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
         self.nodeaddress = self.nodes[0].getnewaddress()
-        self.block_time = time.time() + 1
 
         '''Check that the rules are enforced.'''
         for valid in (True, False):
@@ -77,7 +76,12 @@ class BIP65Test(ComparisonTestFramework):
                 self.invalidate_transaction(spendtx)
                 spendtx.rehash()
 
-            block = create_block(self.tip, create_coinbase(1), self.block_time)
+            gbt = self.nodes[0].getblocktemplate()
+            self.block_time = gbt["mintime"] + 1
+            self.block_bits = int("0x" + gbt["bits"], 0)
+
+            block = create_block(self.tip, create_coinbase(101),
+                                 self.block_time, self.block_bits)
             block.nVersion = 4
             block.vtx.append(spendtx)
             block.hashMerkleRoot = block.calc_merkle_root()
diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py
index 523a694bf..f254843f1 100755
--- a/qa/rpc-tests/bipdersig-p2p.py
+++ b/qa/rpc-tests/bipdersig-p2p.py
@@ -12,7 +12,6 @@ from test_framework.comptool import TestInstance, TestManager
 from test_framework.script import CScript
 from binascii import unhexlify
 import cStringIO
-import time
 
 
 '''
@@ -71,9 +70,9 @@ class BIP66Test(ComparisonTestFramework):
 
     def get_tests(self):
         self.coinbase_blocks = self.nodes[0].generate(1)
+        self.nodes[0].generate(100)
         self.tip = int ("0x" + self.nodes[0].getbestblockhash() + "L", 0)
         self.nodeaddress = self.nodes[0].getnewaddress()
-        self.block_time = time.time() + 1
 
         '''Check that the rules are enforced.'''
         for valid in (True, False):
@@ -84,13 +83,17 @@ class BIP66Test(ComparisonTestFramework):
                 self.invalidate_transaction(spendtx)
                 spendtx.rehash()
 
-            block = create_block(self.tip, create_coinbase(1), self.block_time)
+            gbt = self.nodes[0].getblocktemplate()
+            self.block_time = gbt["mintime"] + 1
+            self.block_bits = int("0x" + gbt["bits"], 0)
+
+            block = create_block(self.tip, create_coinbase(101),
+                                 self.block_time, self.block_bits)
             block.nVersion = 4
             block.vtx.append(spendtx)
             block.hashMerkleRoot = block.calc_merkle_root()
             block.rehash()
             block.solve()
-            self.block_time += 1
             self.tip = block.sha256
             yield TestInstance([[block, valid]])
 

From f8ef223ce381a9f59b1f4b085d360abad6cb9499 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 5 Oct 2017 15:13:28 +0100
Subject: [PATCH 11/11] Un-indent RPC test output in test runner

The indentation caused the test stdout to be buffered and only printed at the
end of the test, which makes it harder to diagnose hanging tests.
---
 qa/pull-tester/rpc-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh
index ed2bfca0a..5631f894f 100755
--- a/qa/pull-tester/rpc-tests.sh
+++ b/qa/pull-tester/rpc-tests.sh
@@ -85,7 +85,7 @@ function runTestScript
 
     echo -e "=== Running testscript ${testName} ==="
 
-    if eval "$@" | sed 's/^/  /'
+    if eval "$@"
     then
         successCount=$(expr $successCount + 1)
         echo "--- Success: ${testName} ---"