Files
dragonx/qa/rpc-tests/test_framework/util.py
DanS c5fde12485 qa: port 12 wallet/mining tests to python3 and fix three framework blockers
Ports qa/rpc-tests from 6 python3 files to 18. Ran every ported test against
the freshly built v1.3.0 dragonxd. Results, honestly:

  PASS (1)            getblocktemplate_proposals.py
  NOT APPLICABLE (7)  wallet.py, walletbackup.py, wallet_protectcoinbase.py,
                      wallet_listnotes.py, wallet_mergetoaddress.py,
                      getblocktemplate.py, wallet_shieldcoinbase.py
  BLOCKED (4)         wallet_sapling, wallet_nullifiers, wallet_persistence,
                      wallet_treestate

The "not applicable" seven are inherited Zcash/Hush-era tests that exercise
features DragonX deliberately removed. They assume transparent t->t value
transfer, but ASSETCHAINS_PRIVATE=1 (hush_utils.h:1826) makes sendtoaddress
and sendmany consensus-refuse; they assume Sprout joinsplits, which are gone
from the RPC layer entirely; and they hardcode Bitcoin economics (10 coin/block,
100-block maturity) against DragonX's 3 DRGX and COINBASE_MATURITY=1. They are
ported and left in place rather than deleted, but they cannot pass on this chain
without being rewritten around z_shieldcoinbase/autoshield.

Three framework fixes in test_framework/util.py, each of which broke every
multi-node test:
  - initialize_chain() passed -connect=0, and init.cpp soft-sets -listen=0 when
    -connect is present, so cache node0 never opened its p2p port and nodes 1-3
    could never sync to it -- initialize_chain() hung forever in sync_blocks().
    Now passes -listen=1 -bind=127.0.0.1 -dnsseed=0 explicitly (an explicit arg
    beats SoftSetBoolArg) while keeping the cache nodes off the public network.
  - cache cleanup removed files from <datadir> when dragonxd writes them one
    level deeper into the net-specific <datadir>/regtest.
  - set_node_times() did print("..." + t) with t an int -> TypeError.
  - default binary paths corrected to src/dragonxd and src/dragonx-cli.

The four BLOCKED tests are blocked by a daemon assert, not by the port; see the
follow-up commit/report on BLOCK_VALID_CONTEXT.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FU87LdsJZiZkfq1eXubpeo
2026-08-31 01:53:43 -05:00

563 lines
22 KiB
Python

# Copyright (c) 2018-2020 The Hush developers
# Copyright (c) 2014 The Bitcoin Core developers
# Copyright (c) 2018-2019 The SuperNET developers
# Distributed under the GPLv3 software license, see the accompanying
# file COPYING or https://www.gnu.org/licenses/gpl-3.0.en.html
# Helpful routines for regression testing
# Add python-bitcoinrpc to module search path:
import os
import sys
from binascii import hexlify, unhexlify
from base64 import b64encode
from decimal import Decimal, ROUND_DOWN
import json
import random
import shutil
import subprocess
import time
import re
from .authproxy import AuthServiceProxy
def p2p_port(n):
return 11000 + n + os.getpid()%999
def rpc_port(n):
return 12000 + n + os.getpid()%999
def rpc_username():
return "hush"
def rpc_password():
return "puppy"
def check_json_precision():
"""Make sure json library being used does not lose precision converting BTC values"""
n = Decimal("20000000.00000003")
satoshis = int(json.loads(json.dumps(float(n)))*1.0e8)
if satoshis != 2000000000000003:
raise RuntimeError("JSON encode/decode loses precision")
def bytes_to_hex_str(byte_str):
return hexlify(byte_str).decode('ascii')
def hex_str_to_bytes(hex_str):
return unhexlify(hex_str.encode('ascii'))
def str_to_b64str(string):
return b64encode(string.encode('utf-8')).decode('ascii')
def sync_blocks(rpc_connections, wait=1):
"""
Wait until everybody has the same block count
"""
while True:
counts = [ x.getblockcount() for x in rpc_connections ]
if counts == [ counts[0] ]*len(counts):
break
time.sleep(wait)
# Now that the block counts are in sync, wait for the internal
# notifications to finish
while True:
notified = [ x.getblockchaininfo()['fullyNotified'] for x in rpc_connections ]
if notified == [ True ] * len(notified):
break
time.sleep(wait)
def sync_mempools(rpc_connections, wait=1):
"""
Wait until everybody has the same transactions in their memory
pools, and has notified all internal listeners of them
"""
while True:
pool = set(rpc_connections[0].getrawmempool())
num_match = 1
for i in range(1, len(rpc_connections)):
if set(rpc_connections[i].getrawmempool()) == pool:
num_match = num_match+1
if num_match == len(rpc_connections):
break
time.sleep(wait)
# Now that the mempools are in sync, wait for the internal
# notifications to finish
while True:
notified = [ x.getmempoolinfo()['fullyNotified'] for x in rpc_connections ]
if notified == [ True ] * len(notified):
break
time.sleep(wait)
bitcoind_processes = {}
def initialize_datadir(dirname, n):
datadir = os.path.join(dirname, "node"+str(n), "regtest")
#datadir = os.path.join(datadir,"ZZZ")
if not os.path.isdir(datadir):
print("Creating dirs %s" % datadir)
os.makedirs(datadir)
print("Writing to " + os.path.join(datadir,"DRAGONX.conf"))
with open(os.path.join(datadir, "DRAGONX.conf"), 'w') as f:
f.write("regtest=1\n");
f.write("txindex=1\n");
#f.write("testnode=1\n");
f.write("server=1\n");
f.write("showmetrics=0\n");
f.write("rpcuser=hush\n");
f.write("rpcpassword=puppy\n");
f.write("port="+str(p2p_port(n))+"\n");
rpcport = str(rpc_port(n))
f.write("rpcport="+rpcport+"\n");
print("RPC port=" + rpcport)
f.write("listenonion=0\n");
# TODO: maybe make these optional, via arg to initialize_datadir, defaulted to on for now
f.write("addressindex=1\n");
f.write("spentindex=1\n");
f.write("timestampindex=1\n");
#f.write("zindex=1\n");
print("Done writing to %s" % os.path.join(datadir,"DRAGONX.conf") )
# dragonxd refuses to start without an asmap file ("Could not find any asmap file!"),
# so every regtest datadir needs one. Link the tree's copy rather than duplicating it.
for src in ("../../../asmap.dat", "../../../src/asmap.dat",
os.path.expanduser("~/.hush/DRAGONX/asmap.dat")):
cand = src if os.path.isabs(src) else os.path.join(os.path.dirname(os.path.abspath(__file__)), src)
if os.path.exists(cand):
dst = os.path.join(datadir, "asmap.dat")
if not os.path.exists(dst):
try: os.symlink(os.path.realpath(cand), dst)
except OSError: shutil.copyfile(cand, dst)
break
return datadir
def initialize_chain(test_dir):
"""
Create (or copy from cache) a 200-block-long chain and
4 wallets.
hushd and hush-cli must be in search path.
"""
print("initialize_chain")
if not os.path.isdir(os.path.join("cache", "node0", "HUSH3")):
devnull = open("/dev/null", "w+")
# Create cache directories, run hushds:
for i in range(4):
datadir=initialize_datadir("cache", i)
# Same two requirements as start_node(): -regtest must be a command-line flag (the
# conf key is ignored, and without it this cache node runs on MAINNET), and -asmap
# must be absolute or dragonxd refuses to start.
# -connect=<anything> makes init.cpp soft-set -listen=0 ("parameter interaction: -connect
# set -> setting -listen=0"), so cache node0 never opened its p2p port and nodes 1-3 could
# never sync to it -- initialize_chain() then hung forever in sync_blocks(). Pass -listen
# and -bind explicitly (an explicit arg beats SoftSetBoolArg) and keep the loopback bind so
# the cache nodes stay off the public network.
args = [ os.getenv("BITCOIND", "src/dragonxd"), "-regtest", "-connect=0", "-keypool=1", "-datadir="+datadir, "-discover=0",
"-listen=1", "-bind=127.0.0.1", "-dnsseed=0" ]
_am = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../asmap.dat")
if os.path.exists(_am):
args.append("-asmap=" + os.path.realpath(_am))
if i > 0:
args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
bitcoind_processes[i] = subprocess.Popen(args)
cmd = os.getenv("BITCOINCLI", "src/dragonx-cli")
cmd_args = cmd + " -datadir="+datadir + " -rpcwait getblockcount"
if os.getenv("PYTHON_DEBUG", ""):
print("initialize_chain: hushd started, calling: " + cmd_args)
strcmd = cmd + " " + "-datadir="+datadir + " -rpcwait getblockcount"
print("Running " + strcmd)
subprocess.check_call(strcmd, shell=True);
#subprocess.check_call([ cmd, "-rpcwait", "getblockcount"], stdout=devnull)
if os.getenv("PYTHON_DEBUG", ""):
print("initialize_chain: hush-cli -rpcwait getblockcount completed")
devnull.close()
rpcs = []
for i in range(4):
try:
url = "http://hush:puppy@127.0.0.1:%d"%(rpc_port(i),)
rpcs.append(AuthServiceProxy(url))
except:
sys.stderr.write("Error connecting to "+url+"\n")
sys.exit(1)
# Create a 200-block-long chain; each of the 4 nodes
# gets 25 mature blocks and 25 immature.
# blocks are created with timestamps 10 minutes apart, starting
# at 1 Jan 2014
# TODO: change this to a recent time
block_time = 1388534400
for i in range(2):
for peer in range(4):
for j in range(25):
set_node_times(rpcs, block_time)
rpcs[peer].generate(1)
# TODO: HUSH3 has 75s blocktime, other HSCs could be different
block_time += 10*75
# Must sync before next peer starts generating blocks
sync_blocks(rpcs)
# Shut them down, and clean up cache directories:
print("Stopping nodes")
stop_nodes(rpcs)
wait_bitcoinds()
for i in range(4):
print("Cleaning up cache dir files")
# log_filename() points at <cache>/node<i>/regtest, but that IS the -datadir we passed;
# dragonxd writes its logs/peers.dat one level deeper, into the net-specific
# <datadir>/regtest subdir (same datadir-vs-netdir split that forced -asmap to be
# absolute in start_node). Try both, and tolerate files a node never created.
for name in ("debug.log", "db.log", "peers.dat", "fee_estimates.dat"):
for cand in (log_filename("cache", i, os.path.join("regtest", name)),
log_filename("cache", i, name)):
try:
os.remove(cand)
except OSError:
pass
for i in range(4):
from_dir = os.path.join("cache", "node"+str(i))
to_dir = os.path.join(test_dir, "node"+str(i))
print("Copying " + from_dir + " to " + to_dir)
shutil.copytree(from_dir, to_dir)
initialize_datadir(test_dir, i) # Overwrite port/rpcport in HUSH3.conf
def initialize_chain_clean(test_dir, num_nodes):
"""
Create an empty blockchain and num_nodes wallets.
Useful if a test case wants complete control over initialization.
"""
for i in range(num_nodes):
initialize_datadir(test_dir, i)
def _rpchost_to_args(rpchost):
'''Convert optional IP:port spec to rpcconnect/rpcport args'''
if rpchost is None:
return []
match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost)
if not match:
raise ValueError('Invalid RPC host spec ' + rpchost)
rpcconnect = match.group(1)
rpcport = match.group(2)
if rpcconnect.startswith('['): # remove IPv6 [...] wrapping
rpcconnect = rpcconnect[1:-1]
rv = ['-rpcconnect=' + rpcconnect]
if rpcport:
rv += ['-rpcport=' + rpcport]
return rv
def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None):
"""
Start a hushd and return RPC connection to it
"""
print("Starting node " + str(i) + " in dir " + dirname)
datadir = os.path.join(dirname, "node"+str(i), "regtest")
if extra_args is None: extra_args = []
# creating special config
if len(extra_args) > 0 and extra_args[0] == '-ac_name=ZZZ':
configpath = datadir + "/DRAGONX.conf"
with open(configpath, "w+") as config:
config.write("rpcuser=hush\n")
config.write("rpcpassword=puppy\n")
port = extra_args[3]
config.write("rpcport=" + (port[9:]) + "\n")
config.write("server=1\n")
#config.write("testnode=1\n")
config.write("txindex=1\n")
config.write("rpcworkqueue=256\n")
config.write("rpcallowip=127.0.0.1\n")
config.write("bind=127.0.0.1\n")
#config.write("listenonion=0\n")
#config.write("torcontrol=127.0.0.1:9999\n")
config.write("rpcbind=127.0.0.1\n")
print("Done writing to %s" % configpath)
if binary is None:
binary = os.getenv("BITCOIND", "src/dragonxd")
# -regtest MUST be a command-line flag. DragonX ignores "regtest=1" in the conf file, so
# without this the node silently runs on MAINNET: it loads the real genesis, dials the real
# seeds and starts syncing the live chain into the test datadir (observed: 196k blocks and
# 679MB before a test timed out). -connect=0 keeps the regtest node off the public network.
args = [ binary, "-regtest", "-connect=0", "-datadir="+datadir, "-keypool=1", "-discover=0", "-rest" ]
# -asmap relative paths are resolved against the NET-SPECIFIC datadir (init.cpp), which for
# regtest is <datadir>/regtest -- so a copy sitting in <datadir> is never found. Pass an
# absolute path; without it dragonxd exits with "Could not find any asmap file!".
_asmap = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../asmap.dat")
if os.path.exists(_asmap):
args.append("-asmap=" + os.path.realpath(_asmap))
if extra_args is not None: args.extend(extra_args)
print("args=" + ' '.join(args))
bitcoind_processes[i] = subprocess.Popen(args)
devnull = open("/dev/null", "w+")
cmd = os.getenv("BITCOINCLI", "src/dragonx-cli")
print("cmd=" + cmd)
# The CLI only needs the datadir: initialize_datadir() already wrote DRAGONX.conf there
# with the right rpcport/user/password. The old form passed extra_args[0] as argv[0] and
# replayed daemon-only flags at the CLI, which only worked for the -ac_name=ZZZ assetchain
# tests and broke every test that passes no extra_args.
args = [ "-regtest", "-datadir="+datadir ]
cmd_args = ' '.join(args) + " -rpcwait getblockcount "
if os.getenv("PYTHON_DEBUG", ""):
print("start_node: hushd started, calling : " + cmd + " " + cmd_args)
strcmd = cmd + " " + cmd_args
print("Running " + strcmd)
import time
time.sleep(2)
subprocess.check_call(strcmd, shell=True);
#subprocess.check_call([ os.getenv("BITCOINCLI", "dragonx-cli"), "-datadir="+datadir] +
# _rpchost_to_args(rpchost) +
# ["-rpcwait", "-rpcport=6438", "getblockcount"], stdout=devnull)
if os.getenv("PYTHON_DEBUG", ""):
print("start_node: calling hush-cli -rpcwait getblockcount returned")
devnull.close()
# Port comes from the same helper initialize_datadir() used, except for the assetchain
# tests which pass it positionally as extra_args[3] == "-rpcport=NNNN".
if len(extra_args) > 3 and str(extra_args[0]) == '-ac_name=ZZZ':
port = extra_args[3][9:]
else:
port = str(rpc_port(i))
#print("port=%s" % port)
username = rpc_username()
password = rpc_password()
url = "http://%s:%s@%s:%s" % (username, password, rpchost or '127.0.0.1', port)
print("connecting to " + url)
if timewait is not None:
proxy = AuthServiceProxy(url, timeout=timewait)
else:
proxy = AuthServiceProxy(url)
#print("created proxy")
proxy.url = url # store URL on proxy for info
return proxy
def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
"""
Start multiple hushds, return RPC connections to them
"""
print("Starting " + str(num_nodes) + " nodes")
if extra_args is None: extra_args = [ None for i in range(num_nodes) ]
if binary is None: binary = [ None for i in range(num_nodes) ]
return [ start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]) for i in range(num_nodes) ]
def log_filename(dirname, n_node, logname):
return os.path.join(dirname, "node"+str(n_node), "regtest", logname)
def check_node(i):
bitcoind_processes[i].poll()
return bitcoind_processes[i].returncode
def stop_node(node, i):
print("Stopping node " + i)
node.stop()
bitcoind_processes[i].wait()
del bitcoind_processes[i]
def stop_nodes(nodes):
for node in nodes:
node.stop()
del nodes[:] # Emptying array closes connections as a side effect
def set_node_times(nodes, t):
print("Setting nodes time to " + str(t))
for node in nodes:
node.setmocktime(t)
def wait_bitcoinds():
print("Waiting for all nodes to cleanly exit")
for bitcoind in bitcoind_processes.values():
bitcoind.wait()
bitcoind_processes.clear()
def connect_nodes(from_connection, node_num):
ip_port = "127.0.0.1:"+str(p2p_port(node_num))
from_connection.addnode(ip_port, "onetry")
# poll until version handshake complete to avoid race conditions
# with transaction relaying
while any(peer['version'] == 0 for peer in from_connection.getpeerinfo()):
time.sleep(0.1)
def connect_nodes_bi(nodes, a, b):
connect_nodes(nodes[a], b)
connect_nodes(nodes[b], a)
def find_output(node, txid, amount):
"""
Return index to output of txid with value amount
Raises exception if there is none.
"""
txdata = node.getrawtransaction(txid, 1)
for i in range(len(txdata["vout"])):
if txdata["vout"][i]["value"] == amount:
return i
raise RuntimeError("find_output txid %s : %s not found"%(txid,str(amount)))
def gather_inputs(from_node, amount_needed, confirmations_required=1):
"""
Return a random set of unspent txouts that are enough to pay amount_needed
"""
assert(confirmations_required >=0)
utxo = from_node.listunspent(confirmations_required)
random.shuffle(utxo)
inputs = []
total_in = Decimal("0.00000000")
while total_in < amount_needed and len(utxo) > 0:
t = utxo.pop()
total_in += t["amount"]
inputs.append({ "txid" : t["txid"], "vout" : t["vout"], "address" : t["address"] } )
if total_in < amount_needed:
raise RuntimeError("Insufficient funds: need %d, have %d"%(amount_needed, total_in))
return (total_in, inputs)
def make_change(from_node, amount_in, amount_out, fee):
"""
Create change output(s), return them
"""
outputs = {}
amount = amount_out+fee
change = amount_in - amount
if change > amount*2:
# Create an extra change output to break up big inputs
change_address = from_node.getnewaddress()
# Split change in two, being careful of rounding:
outputs[change_address] = Decimal(change/2).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
change = amount_in - amount - outputs[change_address]
if change > 0:
outputs[from_node.getnewaddress()] = change
return outputs
def send_zeropri_transaction(from_node, to_node, amount, fee):
"""
Create&broadcast a zero-priority transaction.
Returns (txid, hex-encoded-txdata)
Ensures transaction is zero-priority by first creating a send-to-self,
then using its output
"""
# Create a send-to-self with confirmed inputs:
self_address = from_node.getnewaddress()
(total_in, inputs) = gather_inputs(from_node, amount+fee*2)
outputs = make_change(from_node, total_in, amount+fee, fee)
outputs[self_address] = float(amount+fee)
self_rawtx = from_node.createrawtransaction(inputs, outputs)
self_signresult = from_node.signrawtransaction(self_rawtx)
self_txid = from_node.sendrawtransaction(self_signresult["hex"], True)
vout = find_output(from_node, self_txid, amount+fee)
# Now immediately spend the output to create a 1-input, 1-output
# zero-priority transaction:
inputs = [ { "txid" : self_txid, "vout" : vout } ]
outputs = { to_node.getnewaddress() : float(amount) }
rawtx = from_node.createrawtransaction(inputs, outputs)
signresult = from_node.signrawtransaction(rawtx)
txid = from_node.sendrawtransaction(signresult["hex"], True)
return (txid, signresult["hex"])
def random_zeropri_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
"""
Create a random zero-priority transaction.
Returns (txid, hex-encoded-transaction-data, fee)
"""
from_node = random.choice(nodes)
to_node = random.choice(nodes)
fee = min_fee + fee_increment*random.randint(0,fee_variants)
(txid, txhex) = send_zeropri_transaction(from_node, to_node, amount, fee)
return (txid, txhex, fee)
def random_transaction(nodes, amount, min_fee, fee_increment, fee_variants):
"""
Create a random transaction.
Returns (txid, hex-encoded-transaction-data, fee)
"""
from_node = random.choice(nodes)
to_node = random.choice(nodes)
fee = min_fee + fee_increment*random.randint(0,fee_variants)
(total_in, inputs) = gather_inputs(from_node, amount+fee)
outputs = make_change(from_node, total_in, amount, fee)
outputs[to_node.getnewaddress()] = float(amount)
rawtx = from_node.createrawtransaction(inputs, outputs)
signresult = from_node.signrawtransaction(rawtx)
txid = from_node.sendrawtransaction(signresult["hex"], True)
return (txid, signresult["hex"], fee)
def assert_equal(expected, actual, message=""):
if expected != actual:
if message:
message = "; %s" % message
raise AssertionError("(left == right)%s\n left: <%s>\n right: <%s>" % (message, str(expected), str(actual)))
def assert_true(condition, message = ""):
if not condition:
raise AssertionError(message)
def assert_false(condition, message = ""):
assert_true(not condition, message)
# assert thing2 > thing1
def assert_greater_than(thing1, thing2):
if thing1 <= thing2:
raise AssertionError("%s <= %s"%(str(thing1),str(thing2)))
# assert thing2 >= thing1
def assert_greater_than_or_equal(thing1, thing2):
if thing1 < thing2:
raise AssertionError("%s < %s"%(str(thing1),str(thing2)))
def assert_raises(exc, fun, *args, **kwds):
try:
fun(*args, **kwds)
except exc:
pass
except Exception as e:
raise AssertionError("Unexpected exception raised: "+type(e).__name__)
else:
raise AssertionError("No exception raised")
# Returns txid if operation was a success or None
def wait_and_assert_operationid_status(node, myopid, in_status='success', in_errormsg=None, timeout=300):
print('waiting for async operation {}'.format(myopid))
result = None
for _ in range(1, timeout):
results = node.z_getoperationresult([myopid])
if len(results) > 0:
result = results[0]
break
time.sleep(1)
assert_true(result is not None, "timeout occured")
status = result['status']
txid = None
errormsg = None
if status == "failed":
errormsg = result['error']['message']
elif status == "success":
txid = result['result']['txid']
if os.getenv("PYTHON_DEBUG", ""):
print('...returned status: {}'.format(status))
if errormsg is not None:
print('...returned error: {}'.format(errormsg))
assert_equal(in_status, status, "Operation returned mismatched status. Error Message: {}".format(errormsg))
if errormsg is not None:
assert_true(in_errormsg is not None, "No error retured. Expected: {}".format(errormsg))
assert_true(in_errormsg in errormsg, "Error returned: {}. Error expected: {}".format(errormsg, in_errormsg))
return result # if there was an error return the result
else:
return txid # otherwise return the txid