Auto merge of #2050 - str4d:2020-zmq, r=bitcartel

Add ZeroMQ notifications

Cherry-picked from the following upstream PRs:

- bitcoin/bitcoin#6103
- bitcoin/bitcoin#6684
- bitcoin/bitcoin#6686
- bitcoin/bitcoin#6736
- bitcoin/bitcoin#6739
- bitcoin/bitcoin#6743
- bitcoin/bitcoin#6768
- bitcoin/bitcoin#6779
- bitcoin/bitcoin#6810
- bitcoin/bitcoin#6927
- bitcoin/bitcoin#6980 (only upgrading zeromq)
- bitcoin/bitcoin#6680
- bitcoin/bitcoin#7058
- bitcoin/bitcoin#7621
- bitcoin/bitcoin#7335 (only parts affecting `zmq_test.py`)
- bitcoin/bitcoin#7853 (only parts affecting `zmq_test.py`)
- bitcoin/bitcoin#7762
- bitcoin/bitcoin#7993 (only upgrading zeromq)
- bitcoin/bitcoin#8238
- bitcoin/bitcoin#8701
- bitcoin/bitcoin#6685

Closes #2020.

qa/pull-tester/rpc-tests.sh

@@ -57,6 +57,10 @@ testScriptsExt=(
     'p2p-acceptblock.py'
 );
 
+if [ "x$ENABLE_ZMQ" = "x1" ]; then
+  testScripts+=('zmq_test.py')
+fi
+
 extArg="-extended"
 passOn=${@#$extArg}
 

qa/pull-tester/tests-config.sh.in

@@ -10,6 +10,7 @@ EXEEXT="@EXEEXT@"
 @ENABLE_WALLET_TRUE@ENABLE_WALLET=1
 @BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1
+@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1
 
 REAL_BITCOIND="$BUILDDIR/src/zcashd${EXEEXT}"
 REAL_BITCOINCLI="$BUILDDIR/src/zcash-cli${EXEEXT}"

qa/rpc-tests/test_framework/util.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2014 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+
 #
 # Helpful routines for regression testing
 #
@@ -9,6 +11,8 @@
 import os
 import sys
 
+from binascii import hexlify, unhexlify
+from base64 import b64encode
 from decimal import Decimal, ROUND_DOWN
 import json
 import random
@@ -32,6 +36,15 @@ def check_json_precision():
     if satoshis != 2000000000000003:
         raise RuntimeError("JSON encode/decode loses precision")
 
+def bytes_to_hex_str(byte_str):
+    return hexlify(byte_str).decode('ascii')
+
+def hex_str_to_bytes(hex_str):
+    return unhexlify(hex_str.encode('ascii'))
+
+def str_to_b64str(string):
+    return b64encode(string.encode('utf-8')).decode('ascii')
+
 def sync_blocks(rpc_connections, wait=1):
     """
     Wait until everybody has the same block count

qa/rpc-tests/zmq_test.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python2
+# Copyright (c) 2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test ZMQ interface
+#
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+import zmq
+import binascii
+import struct
+
+try:
+    import http.client as httplib
+except ImportError:
+    import httplib
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urlparse
+
+class ZMQTest (BitcoinTestFramework):
+
+    port = 28332
+
+    def setup_nodes(self):
+        self.zmqContext = zmq.Context()
+        self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
+        self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock")
+        self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx")
+        self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port)
+        return start_nodes(4, self.options.tmpdir, extra_args=[
+            ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)],
+            [],
+            [],
+            []
+            ])
+
+    def run_test(self):
+        self.sync_all()
+
+        genhashes = self.nodes[0].generate(1)
+        self.sync_all()
+
+        print "listen..."
+        msg = self.zmqSubSocket.recv_multipart()
+        topic = msg[0]
+        assert_equal(topic, b"hashtx")
+        body = msg[1]
+        nseq = msg[2]
+        msgSequence = struct.unpack('<I', msg[-1])[-1]
+        assert_equal(msgSequence, 0) #must be sequence 0 on hashtx
+
+        msg = self.zmqSubSocket.recv_multipart()
+        topic = msg[0]
+        body = msg[1]
+        msgSequence = struct.unpack('<I', msg[-1])[-1]
+        assert_equal(msgSequence, 0) #must be sequence 0 on hashblock
+        blkhash = bytes_to_hex_str(body)
+
+        assert_equal(genhashes[0], blkhash) #blockhash from generate must be equal to the hash received over zmq
+
+        n = 10
+        genhashes = self.nodes[1].generate(n)
+        self.sync_all()
+
+        zmqHashes = []
+        blockcount = 0
+        for x in range(0,n*2):
+            msg = self.zmqSubSocket.recv_multipart()
+            topic = msg[0]
+            body = msg[1]
+            if topic == b"hashblock":
+                zmqHashes.append(bytes_to_hex_str(body))
+                msgSequence = struct.unpack('<I', msg[-1])[-1]
+                assert_equal(msgSequence, blockcount+1)
+                blockcount += 1
+
+        for x in range(0,n):
+            assert_equal(genhashes[x], zmqHashes[x]) #blockhash from generate must be equal to the hash received over zmq
+
+        #test tx from a second node
+        hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0)
+        self.sync_all()
+
+        # now we should receive a zmq msg because the tx was broadcast
+        msg = self.zmqSubSocket.recv_multipart()
+        topic = msg[0]
+        body = msg[1]
+        hashZMQ = ""
+        if topic == b"hashtx":
+            hashZMQ = bytes_to_hex_str(body)
+            msgSequence = struct.unpack('<I', msg[-1])[-1]
+            assert_equal(msgSequence, blockcount+1)
+
+        assert_equal(hashRPC, hashZMQ) #blockhash from generate must be equal to the hash received over zmq
+
+
+if __name__ == '__main__':
+    ZMQTest ().main ()