feat(merge): run per-address consolidation as a paced background job
"Keep in each address" posted one task that called z_mergetoaddress for every batch of every address back to back; each call is a whole-wallet scan under cs_main, and the daemon runs the ops one at a time anyway. It is now an App-owned services::ConsolidationRun: - one RPC at a time; the next merge only after the previous batch left the queue and a gap as long as the last call took (2-120 s); - pauses while the node syncs, the wallet is locked, a send of ours is in progress, or another wallet scan is running; - failures classified: nothing that may have queued is ever resent; a breaker stops after 3 dead addresses or 3 failed ops in a row; - each merge is charged to the scan budget; one re-read and one forced balance at the end; - it keeps running when the dialog closes: a status-bar chip, a run view with progress, the wait reason and a Stop button. The "Into one" merge can no longer be confirmed twice while its call is in flight. fake_dragonxd gains FAKE_SCAN_SECS, FAKE_MERGE_SERIAL and FAKE_MERGE_FAIL. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -4521,6 +4521,10 @@ void testConsolidationStringsTranslated()
|
||||
"merge_keep_each_status_done", "merge_keep_each_status_running", "merge_keep_each_run_again",
|
||||
"merge_waiting_node", "merge_wait_locked", "merge_scope_failed_fmt", "retry",
|
||||
"merge_keep_each_status_unconfirmed", "merge_submit_uncertain",
|
||||
"merge_run_progress_fmt", "merge_run_stop", "merge_run_stopping", "merge_run_status_waiting",
|
||||
"merge_run_status_not_run", "merge_run_wait_sync", "merge_run_wait_send", "merge_run_wait_batch",
|
||||
"merge_run_err_timeout", "merge_run_stopped_toast", "merge_run_aborted_toast", "merge_run_start_failed",
|
||||
"merge_run_chip_fmt", "merge_run_halted_fmt", "merge_run_busy", "merge_run_migration_pending",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -11613,6 +11617,613 @@ void testChatScanGovernorStreams()
|
||||
EXPECT_TRUE(static_cast<bool>(gov.tryAdmit(ScanStream::Balance, true, a, 10.0)));
|
||||
}
|
||||
|
||||
// C210: reading z_mergetoaddress's reply, and one call into an address's run.
|
||||
void testMergeCallResult()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
using nlohmann::json;
|
||||
MergeCallResult r;
|
||||
readMergeCallResult(json{{"opid", "o"}, {"mergingNotes", 50}, {"remainingNotes", 70}}, r);
|
||||
EXPECT_TRUE(r.opid == "o" && r.merging == 50 && r.remaining == 70 && r.error.empty());
|
||||
readMergeCallResult(json{{"opid", "o"}, {"mergingNotes", "50"}}, r);
|
||||
EXPECT_TRUE(r.opid == "o" && r.merging == 0 && r.error.empty()); // accepted, count unreadable
|
||||
for (const json& bad : {json{{"mergingNotes", 3}}, json{{"opid", 5}}, json{{"opid", ""}}, json::array(), json("x"), json()}) {
|
||||
readMergeCallResult(bad, r);
|
||||
EXPECT_TRUE(r.error == "no operation id" && r.opid.empty());
|
||||
}
|
||||
readMergeCallResult(json{{"opid", "o"}, {"remainingNotes", -4}}, r);
|
||||
EXPECT_EQ(r.remaining, 0);
|
||||
readMergeCallResult(json{{"opid", "o"}, {"remainingNotes", static_cast<std::uint64_t>(1) << 40}}, r);
|
||||
EXPECT_EQ(r.remaining, INT_MAX);
|
||||
|
||||
AddressRun run;
|
||||
MergeCallResult ok{"o1", 50, 70, ""};
|
||||
EXPECT_TRUE(recordMergeResult(run, ok, 20));
|
||||
EXPECT_FALSE(recordMergeResult(run, MergeCallResult{"o2", 50, 1, ""}, 20)); // one left: done
|
||||
AddressRun capped;
|
||||
EXPECT_FALSE(recordMergeResult(capped, ok, 1)); // at maxBatches
|
||||
AddressRun failed;
|
||||
EXPECT_FALSE(recordMergeResult(failed, MergeCallResult{"", 0, 0, "boom"}, 20));
|
||||
EXPECT_EQ(failed.error, std::string("boom"));
|
||||
int calls = 0;
|
||||
consolidateInPlace("z", [&](const std::string&) { ++calls; return ok; }, 0);
|
||||
EXPECT_EQ(calls, 0);
|
||||
}
|
||||
|
||||
void testOpReports()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
using nlohmann::json;
|
||||
EXPECT_TRUE(parseOpReports(json::object()).empty());
|
||||
const json reply = json::array({
|
||||
json(5),
|
||||
json{{"status", "success"}},
|
||||
json{{"id", 7}, {"status", "success"}},
|
||||
json{{"id", ""}, {"status", "success"}},
|
||||
json{{"id", "a"}, {"status", 3}, {"error", "x"}, {"result", "y"}},
|
||||
json{{"id", "b"}, {"status", "failed"}, {"error", json{{"message", "bad"}}}},
|
||||
json{{"id", "c"}, {"status", "success"}, {"result", json{{"txid", "tx-c"}}}},
|
||||
});
|
||||
auto all = parseOpReports(reply);
|
||||
EXPECT_EQ(all.size(), static_cast<size_t>(3));
|
||||
EXPECT_TRUE(all[0].id == "a" && all[0].status.empty() && all[0].error.empty() && all[0].txid.empty());
|
||||
EXPECT_EQ(all[1].error, std::string("bad"));
|
||||
EXPECT_EQ(all[2].txid, std::string("tx-c"));
|
||||
auto wanted = parseOpReports(reply, {"c"});
|
||||
EXPECT_TRUE(wanted.size() == 1 && wanted[0].id == "c");
|
||||
EXPECT_TRUE(opStateFromStatus("queued") == OpState::Queued);
|
||||
EXPECT_TRUE(opStateFromStatus("executing") == OpState::Executing);
|
||||
EXPECT_TRUE(opStateFromStatus("success") == OpState::Success);
|
||||
for (const char* f : {"failed", "cancelled", "canceled"}) EXPECT_TRUE(opStateFromStatus(f) == OpState::Failed);
|
||||
EXPECT_TRUE(opStateFromStatus("") == OpState::Unknown);
|
||||
EXPECT_TRUE(opStateFromStatus("x") == OpState::Unknown);
|
||||
}
|
||||
|
||||
void testMergeFailureClassification()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
using K = dragonx::rpc::RpcFailureKind;
|
||||
EXPECT_TRUE(classifyMergeFailure(K::None, "") == MergeFailure::Uncertain);
|
||||
for (K k : {K::Deferred, K::NotListening, K::ConnectTimeout, K::Warmup, K::QueueFull})
|
||||
EXPECT_TRUE(classifyMergeFailure(k, "x") == MergeFailure::NotSent);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::ReplyTimeout, "x") == MergeFailure::Uncertain);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::Aborted, "x") == MergeFailure::Uncertain);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::ShuttingDown, "x") == MergeFailure::Transient);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::Unhandled, "x") == MergeFailure::Transient);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::Auth, "x") == MergeFailure::Fatal);
|
||||
EXPECT_TRUE(classifyMergeFailure(K::Other, "Not connected") == MergeFailure::NotSent);
|
||||
// By kind and message alone, an unclassified error is fatal...
|
||||
EXPECT_TRUE(classifyMergeFailure(K::Other, "RPC request failed: Failure when receiving data from the peer") == MergeFailure::Fatal);
|
||||
{ // ...but a call that was sent and then cut off before any HTTP status may have queued the merge.
|
||||
using dragonx::rpc::RpcTransportError;
|
||||
const RpcTransportError cut(K::Other, "RPC request failed: Failure when receiving data from the peer", CURLE_RECV_ERROR, 0, true);
|
||||
EXPECT_TRUE(mergeCallFailureKind(cut) == K::ReplyTimeout);
|
||||
EXPECT_TRUE(classifyMergeFailure(mergeCallFailureKind(cut), cut.what()) == MergeFailure::Uncertain);
|
||||
const RpcTransportError unsent(K::Other, "RPC request failed: x", CURLE_RECV_ERROR, 0, false);
|
||||
EXPECT_TRUE(mergeCallFailureKind(unsent) == K::Other);
|
||||
const RpcTransportError http(K::Other, "RPC error: HTTP 504", CURLE_OK, 504, true);
|
||||
EXPECT_TRUE(mergeCallFailureKind(http) == K::Other);
|
||||
EXPECT_TRUE(mergeCallFailureKind(std::runtime_error("Not connected")) == K::Other);
|
||||
EXPECT_TRUE(classifyMergeFailure(mergeCallFailureKind(std::runtime_error("Not connected")), "Not connected") == MergeFailure::NotSent);
|
||||
const RpcTransportError aborted(K::Aborted, "RPC aborted: disconnecting", CURLE_OK, 0, false);
|
||||
EXPECT_TRUE(mergeCallFailureKind(aborted) == K::Aborted);
|
||||
}
|
||||
const std::pair<const char*, MergeFailure> cases[] = {
|
||||
{"z_mergetoaddress: Extreme Privacy! Chain still syncing at height 5, aborting to prevent linkability analysis. Please wait until FULLY SYNCED and try again.", MergeFailure::Syncing},
|
||||
{"Could not find spending key for payment address.", MergeFailure::NeedsUnlock},
|
||||
{"Error: Please enter the wallet passphrase with walletpassphrase first.", MergeFailure::NeedsUnlock},
|
||||
{"Could not find any funds to merge.", MergeFailure::NothingLeft},
|
||||
{"Destination address is also the only source address, and all its funds are already merged.", MergeFailure::NothingLeft},
|
||||
{"Insufficient funds, have 0.00005, which is less than miners fee 0.0001", MergeFailure::NothingLeft},
|
||||
{"Fee 0.0001 is greater than the net amount to be shielded 0.00005", MergeFailure::NothingLeft},
|
||||
{"Insufficient funds to pay the fee", MergeFailure::NothingLeft},
|
||||
{"Async RPC queue is shutting down; the operation was not queued", MergeFailure::Transient},
|
||||
{"Invalid parameter, unknown address format", MergeFailure::Fatal},
|
||||
};
|
||||
for (const auto& [msg, want] : cases) EXPECT_TRUE(classifyMergeFailure(K::AppError, msg) == want);
|
||||
|
||||
EXPECT_TRUE(runWaitKey(RunWait::None) == nullptr && runWaitKey(RunWait::InFlight) == nullptr);
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Node)), std::string("merge_waiting_node"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Busy)), std::string("merge_waiting_node"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Syncing)), std::string("merge_run_wait_sync"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Locked)), std::string("merge_wait_locked"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::UserSend)), std::string("merge_run_wait_send"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Scan)), std::string("merge_run_wait_batch"));
|
||||
EXPECT_EQ(std::string(runWaitKey(RunWait::Batch)), std::string("merge_run_wait_batch"));
|
||||
RunSummary sum;
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::None);
|
||||
sum.end = RunEnd::Stopped; EXPECT_TRUE(runVerdict(sum) == RunVerdict::Stopped);
|
||||
sum.end = RunEnd::Halted; EXPECT_TRUE(runVerdict(sum) == RunVerdict::Halted);
|
||||
sum.end = RunEnd::Aborted; EXPECT_TRUE(runVerdict(sum) == RunVerdict::Aborted);
|
||||
sum.end = RunEnd::Completed;
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::NothingSubmitted);
|
||||
sum.batchesSubmitted = 3;
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::Clean);
|
||||
sum.addressesUnconfirmed = 1;
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::Unconfirmed);
|
||||
sum.addressesFailed = 1;
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::SomeFailed);
|
||||
}
|
||||
|
||||
namespace {
|
||||
dragonx::services::RunStep stepRun(dragonx::services::ConsolidationRun& run, const dragonx::services::RunEnv& env, double now)
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
const RunStep st = run.next(env, now);
|
||||
if (st.action == RunAction::Call) run.markCallPosted(st.addrIdx, now);
|
||||
else if (st.action == RunAction::Poll) run.markPollPosted(now);
|
||||
return st;
|
||||
}
|
||||
dragonx::services::OpReport opRep(const std::string& id, const std::string& status, const std::string& error = "",
|
||||
const std::string& txid = "")
|
||||
{
|
||||
dragonx::services::OpReport r;
|
||||
r.id = id;
|
||||
r.status = status;
|
||||
r.error = error;
|
||||
r.txid = txid;
|
||||
return r;
|
||||
}
|
||||
dragonx::services::RunEnv okRunEnv()
|
||||
{
|
||||
dragonx::services::RunEnv env;
|
||||
env.nodeUsable = true;
|
||||
return env;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void testConsolidationRun()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
using K = dragonx::rpc::RpcFailureKind;
|
||||
const RunEnv ok = okRunEnv();
|
||||
{ // Start and dismiss.
|
||||
ConsolidationRun run;
|
||||
EXPECT_FALSE(run.start({}, RunConfig{}, 0.0));
|
||||
EXPECT_FALSE(run.start({{"z1", 1, 5.0}}, RunConfig{}, 0.0)); // one note
|
||||
EXPECT_FALSE(run.start({{"z1", 5, 0.0001}}, RunConfig{}, 0.0)); // too small for the fee
|
||||
RunConfig small;
|
||||
small.limit = 1;
|
||||
EXPECT_TRUE(run.start({{"z1", 5, 5.0}}, small, 0.0));
|
||||
EXPECT_EQ(run.config().limit, 2);
|
||||
EXPECT_FALSE(run.start({{"z2", 5, 5.0}}, RunConfig{}, 0.0)); // already running
|
||||
run.dismiss();
|
||||
EXPECT_TRUE(run.active()); // no-op while running
|
||||
const auto gen1 = run.gen();
|
||||
run.requestStop();
|
||||
EXPECT_TRUE(stepRun(run, okRunEnv(), 1.0).action == RunAction::Finish);
|
||||
EXPECT_TRUE(run.end() == RunEnd::Stopped);
|
||||
run.dismiss();
|
||||
EXPECT_FALSE(run.hasRun());
|
||||
EXPECT_EQ(run.gen(), gen1); // kept: stale results never match later
|
||||
EXPECT_TRUE(run.start({{"z1", 5, 5.0}}, RunConfig{}, 2.0));
|
||||
EXPECT_TRUE(run.gen() > gen1);
|
||||
}
|
||||
{ // A normal run: call, poll, call after the gap and the op gate, ..., one Finish.
|
||||
ConsolidationRun run;
|
||||
EXPECT_TRUE(run.start({{"zA", 120, 60.0}}, RunConfig{}, 0.0));
|
||||
auto st = stepRun(run, ok, 0.0);
|
||||
EXPECT_TRUE(st.action == RunAction::Call && st.addrIdx == 0);
|
||||
EXPECT_TRUE(stepRun(run, ok, 0.5).action == RunAction::None); // one RPC at a time
|
||||
EXPECT_TRUE(run.lastWait() == RunWait::InFlight);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o1", 50, 70, ""}, K::None, 1.0, 1.0);
|
||||
EXPECT_TRUE(stepRun(run, ok, 1.0).action == RunAction::Poll); // learn its state first
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "queued")}, 1.1);
|
||||
EXPECT_TRUE(stepRun(run, ok, 3.5).action == RunAction::None); // past the gap, but still queued
|
||||
EXPECT_TRUE(run.lastWait() == RunWait::Batch);
|
||||
EXPECT_TRUE(stepRun(run, ok, 4.1).action == RunAction::Poll);
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, 4.2);
|
||||
st = stepRun(run, ok, 4.3);
|
||||
EXPECT_TRUE(st.action == RunAction::Call);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o2", 50, 20, ""}, K::None, 1.0, 5.0);
|
||||
EXPECT_TRUE(stepRun(run, ok, 5.0).action == RunAction::Poll);
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "success", "", "tx1"), opRep("o2", "executing")}, 5.1);
|
||||
EXPECT_TRUE(stepRun(run, ok, 7.0).action == RunAction::Call);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o3", 20, 0, ""}, K::None, 1.0, 8.0);
|
||||
EXPECT_TRUE(run.addresses()[0].done);
|
||||
double t = 8.0;
|
||||
int finishes = 0;
|
||||
for (; t < 60.0; t += 0.5) {
|
||||
const auto s2 = stepRun(run, ok, t);
|
||||
if (s2.action == RunAction::Poll)
|
||||
run.onPollResult(run.gen(), true, {opRep("o2", "success"), opRep("o3", "success")}, t + 0.1);
|
||||
if (s2.action == RunAction::Finish) ++finishes;
|
||||
EXPECT_TRUE(s2.action != RunAction::Call);
|
||||
}
|
||||
EXPECT_EQ(finishes, 1);
|
||||
EXPECT_TRUE(run.end() == RunEnd::Completed);
|
||||
const auto sum = run.summary();
|
||||
EXPECT_EQ(sum.batchesSucceeded, 3);
|
||||
EXPECT_EQ(sum.notesMerged, 120);
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::Clean);
|
||||
EXPECT_TRUE(run.next(ok, 70.0).action == RunAction::None);
|
||||
}
|
||||
{ // The gap is the call's own cost, within [2 s, 120 s].
|
||||
for (const auto& [callSec, gap] : {std::pair<double, double>{0.1, 2.0}, {15.0, 15.0}, {200.0, 120.0}}) {
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, callSec, 1.0);
|
||||
// Advance to `t`, answering polls (the op stays Executing); the first Call's time.
|
||||
double firstCall = -1.0;
|
||||
for (double t = 1.0; t <= 1.0 + gap + 1.0 && firstCall < 0.0; t += 0.01) {
|
||||
RunStep s2 = stepRun(run, ok, t);
|
||||
while (s2.action == RunAction::Poll) { // a due poll goes first
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, t);
|
||||
s2 = stepRun(run, ok, t);
|
||||
}
|
||||
if (s2.action == RunAction::Call) firstCall = t;
|
||||
}
|
||||
EXPECT_NEAR(firstCall, 1.0 + gap, 0.02);
|
||||
}
|
||||
}
|
||||
{ // NotSent never ends the address; each retry is at least 5 s later.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
double t = 0.0, lastCall = -100.0;
|
||||
int calls = 0;
|
||||
while (calls < 50 && t < 1000.0) {
|
||||
const auto s2 = stepRun(run, ok, t);
|
||||
if (s2.action == RunAction::Call) {
|
||||
EXPECT_TRUE(t - lastCall >= 5.0);
|
||||
lastCall = t;
|
||||
++calls;
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "held back"}, K::Deferred, 0.01, t);
|
||||
}
|
||||
t += 0.5;
|
||||
}
|
||||
EXPECT_EQ(calls, 50);
|
||||
EXPECT_FALSE(run.addresses()[0].done);
|
||||
}
|
||||
{ // Syncing holds 30 s; NeedsUnlock holds 30 s and asks for one lock re-read, then fails the address.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10, 5.0}, {"zB", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "Chain still syncing at height 5"}, K::AppError, 0.1, 0.1);
|
||||
EXPECT_TRUE(stepRun(run, ok, 29.0).action == RunAction::None);
|
||||
EXPECT_TRUE(run.lastWait() == RunWait::Syncing);
|
||||
EXPECT_TRUE(run.addresses()[0].transient == 0 && run.addresses()[0].unlockRefusals == 0);
|
||||
EXPECT_TRUE(stepRun(run, ok, 30.2).action == RunAction::Call);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "Could not find spending key for payment address."},
|
||||
K::AppError, 0.1, 30.3);
|
||||
EXPECT_TRUE(run.consumeLockResyncRequest());
|
||||
EXPECT_FALSE(run.consumeLockResyncRequest());
|
||||
EXPECT_TRUE(stepRun(run, ok, 59.0).action == RunAction::None);
|
||||
EXPECT_TRUE(run.lastWait() == RunWait::Locked);
|
||||
EXPECT_TRUE(stepRun(run, ok, 60.5).action == RunAction::Call);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "Could not find spending key for payment address."},
|
||||
K::AppError, 0.1, 60.6);
|
||||
EXPECT_TRUE(run.addresses()[0].done && run.addresses()[0].outcome == AddrOutcome::Failed);
|
||||
const auto s3 = stepRun(run, ok, 63.0);
|
||||
EXPECT_TRUE(s3.action == RunAction::Call && s3.addrIdx == 1);
|
||||
}
|
||||
{ // NothingLeft: Done after a batch, Failed without one. Uncertain is never resent.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10, 5.0}, {"zB", 10, 5.0}, {"zC", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o1", 5, 5, ""}, K::None, 0.1, 0.1);
|
||||
stepRun(run, ok, 0.2);
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, 0.2);
|
||||
EXPECT_TRUE(stepRun(run, ok, 2.5).action == RunAction::Call);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "Could not find any funds to merge."}, K::AppError, 0.1, 2.6);
|
||||
EXPECT_TRUE(run.addresses()[0].outcome == AddrOutcome::Done);
|
||||
double t = 2.6;
|
||||
RunStep s2;
|
||||
do { t += 0.5; s2 = stepRun(run, ok, t); if (s2.action == RunAction::Poll) run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, t); }
|
||||
while (s2.action != RunAction::Call && t < 100.0);
|
||||
EXPECT_EQ(s2.addrIdx, 1);
|
||||
run.onCallResult(run.gen(), 1, MergeCallResult{"", 0, 0, "Could not find any funds to merge."}, K::AppError, 0.1, t);
|
||||
EXPECT_TRUE(run.addresses()[1].outcome == AddrOutcome::Failed);
|
||||
do { t += 0.5; s2 = stepRun(run, ok, t); if (s2.action == RunAction::Poll) run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, t); }
|
||||
while (s2.action != RunAction::Call && t < 200.0);
|
||||
EXPECT_EQ(s2.addrIdx, 2);
|
||||
run.onCallResult(run.gen(), 2, MergeCallResult{"", 0, 0, "RPC error: reply timeout"}, K::ReplyTimeout, 120.0, t);
|
||||
EXPECT_TRUE(run.addresses()[2].done && run.addresses()[2].outcome == AddrOutcome::Uncertain);
|
||||
for (int i = 0; i < 400; ++i) {
|
||||
t += 0.5;
|
||||
s2 = stepRun(run, ok, t);
|
||||
EXPECT_TRUE(s2.action != RunAction::Call);
|
||||
if (s2.action == RunAction::Poll) run.onPollResult(run.gen(), true, {opRep("o1", "success")}, t);
|
||||
}
|
||||
EXPECT_TRUE(run.end() == RunEnd::Completed);
|
||||
const auto sum = run.summary();
|
||||
EXPECT_EQ(sum.addressesUnconfirmed, 1);
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::SomeFailed); // zB failed (worse than unconfirmed)
|
||||
}
|
||||
{ // Breakers: three dead addresses in a row halt the run; an accepted call between resets the streak.
|
||||
ConsolidationRun run;
|
||||
run.start({{"z1", 10, 5.0}, {"z2", 10, 5.0}, {"z3", 10, 5.0}, {"z4", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
double t = 0.0;
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
RunStep s2;
|
||||
do { s2 = stepRun(run, ok, t); t += 0.5; } while (s2.action != RunAction::Call);
|
||||
run.onCallResult(run.gen(), s2.addrIdx, MergeCallResult{"", 0, 0, "Invalid parameter, unknown address format"},
|
||||
K::AppError, 0.1, t);
|
||||
}
|
||||
RunStep fin;
|
||||
do { fin = stepRun(run, ok, t); t += 0.5; } while (fin.action == RunAction::None && t < 100.0);
|
||||
EXPECT_TRUE(fin.action == RunAction::Finish && run.end() == RunEnd::Halted);
|
||||
EXPECT_FALSE(run.summary().firstError.empty());
|
||||
EXPECT_FALSE(run.addresses()[3].done); // never tried
|
||||
}
|
||||
{ // Op-level breaker: three failed ops in a row.
|
||||
ConsolidationRun run;
|
||||
run.start({{"z1", 10, 5.0}, {"z2", 10, 5.0}, {"z3", 10, 5.0}, {"z4", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
double t = 0.0;
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
RunStep s2;
|
||||
do {
|
||||
s2 = stepRun(run, ok, t);
|
||||
if (s2.action == RunAction::Poll) {
|
||||
std::vector<OpReport> reps;
|
||||
for (const auto& id : run.outstandingOpids()) reps.push_back(opRep(id, "failed", "proof failed"));
|
||||
run.onPollResult(run.gen(), true, reps, t);
|
||||
}
|
||||
t += 0.5;
|
||||
} while (s2.action != RunAction::Call && t < 500.0);
|
||||
if (s2.action == RunAction::Call)
|
||||
run.onCallResult(run.gen(), s2.addrIdx, MergeCallResult{"op" + std::to_string(i), 9, 1, ""}, K::None, 0.1, t);
|
||||
}
|
||||
RunStep last;
|
||||
for (int i = 0; i < 100 && last.action != RunAction::Finish; ++i) {
|
||||
last = stepRun(run, ok, t);
|
||||
if (last.action == RunAction::Poll) {
|
||||
std::vector<OpReport> reps;
|
||||
for (const auto& id : run.outstandingOpids()) reps.push_back(opRep(id, "failed", "proof failed"));
|
||||
run.onPollResult(run.gen(), true, reps, t);
|
||||
}
|
||||
t += 0.5;
|
||||
}
|
||||
EXPECT_TRUE(run.end() == RunEnd::Halted);
|
||||
EXPECT_FALSE(run.addresses()[3].done);
|
||||
}
|
||||
{ // Transient: backoff 5, 10, 20, 40, 60 s, then the address fails.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
double t = 0.0, last = 0.0;
|
||||
const double expect[] = {5.0, 10.0, 20.0, 40.0, 60.0};
|
||||
for (int i = 0; i <= 5; ++i) {
|
||||
RunStep s2;
|
||||
do { s2 = stepRun(run, ok, t); if (s2.action != RunAction::Call) t += 0.25; } while (s2.action != RunAction::Call && t < 1000.0);
|
||||
if (i > 0) EXPECT_TRUE(t - last >= expect[i - 1] - 0.01 && t - last <= expect[i - 1] + 0.3);
|
||||
last = t;
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"", 0, 0, "HTTP 503"}, K::ShuttingDown, 0.0, t);
|
||||
}
|
||||
EXPECT_TRUE(run.addresses()[0].done && run.addresses()[0].outcome == AddrOutcome::Failed);
|
||||
}
|
||||
{ // A result for another address or generation is an orphan: returned for tracking, state unchanged.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10, 5.0}, {"zB", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
EXPECT_EQ(run.onCallResult(run.gen(), 1, MergeCallResult{"oX", 5, 5, ""}, K::None, 0.1, 0.1), std::string("oX"));
|
||||
EXPECT_EQ(run.onCallResult(run.gen() + 7, 0, MergeCallResult{"oY", 5, 5, ""}, K::None, 0.1, 0.1), std::string("oY"));
|
||||
EXPECT_TRUE(run.batches().empty());
|
||||
EXPECT_TRUE(stepRun(run, ok, 0.2).action == RunAction::None); // the real call is still in flight
|
||||
}
|
||||
{ // Environment gates: Node and Busy hold calls and polls; Syncing, Locked, UserSend and Scan hold only calls.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 200, 100.0}, {"zB", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
RunEnv node = ok; node.nodeUsable = false;
|
||||
RunEnv busy = ok; busy.busy = true;
|
||||
EXPECT_TRUE(stepRun(run, node, 1.0).action == RunAction::None && run.lastWait() == RunWait::Node);
|
||||
EXPECT_TRUE(stepRun(run, busy, 1.0).action == RunAction::None && run.lastWait() == RunWait::Busy);
|
||||
RunEnv syncing = ok; syncing.callsAllowed = false;
|
||||
EXPECT_TRUE(stepRun(run, syncing, 1.0).action == RunAction::Poll); // a due poll still runs
|
||||
run.onPollResult(run.gen(), true, {opRep("o1", "executing")}, 1.0);
|
||||
EXPECT_TRUE(stepRun(run, syncing, 3.0).action == RunAction::None && run.lastWait() == RunWait::Syncing);
|
||||
RunEnv locked = ok; locked.locked = true;
|
||||
EXPECT_TRUE(stepRun(run, locked, 3.0).action == RunAction::None && run.lastWait() == RunWait::Locked);
|
||||
RunEnv send = ok; send.userSendBusy = true;
|
||||
EXPECT_TRUE(stepRun(run, send, 3.0).action == RunAction::None && run.lastWait() == RunWait::UserSend);
|
||||
RunEnv scan = ok; scan.scanBusy = true;
|
||||
EXPECT_TRUE(stepRun(run, scan, 3.0).action == RunAction::None && run.lastWait() == RunWait::Scan);
|
||||
EXPECT_TRUE(stepRun(run, ok, 3.0).action == RunAction::Call);
|
||||
}
|
||||
{ // Stop: the call in flight is recorded, no further call, polls until terminal, then Stopped.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 200, 100.0}, {"zB", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
run.requestStop();
|
||||
run.onCallResult(run.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
double t = 0.1;
|
||||
RunStep s2;
|
||||
for (int i = 0; i < 100 && s2.action != RunAction::Finish; ++i) {
|
||||
t += 0.5;
|
||||
s2 = stepRun(run, ok, t);
|
||||
EXPECT_TRUE(s2.action != RunAction::Call);
|
||||
if (s2.action == RunAction::Poll) run.onPollResult(run.gen(), true, {opRep("o1", i < 4 ? "executing" : "success")}, t);
|
||||
}
|
||||
EXPECT_TRUE(run.end() == RunEnd::Stopped);
|
||||
ConsolidationRun idle; // nothing outstanding, node gone: finishes at once
|
||||
idle.start({{"zA", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
idle.requestStop();
|
||||
RunEnv gone = ok; gone.nodeUsable = false;
|
||||
EXPECT_TRUE(idle.next(gone, 1.0).action == RunAction::Finish);
|
||||
}
|
||||
{ // abort(): false unless running; the outstanding batch is Lost; late results are orphans / ignored.
|
||||
ConsolidationRun run;
|
||||
EXPECT_FALSE(run.abort());
|
||||
run.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
const auto gen = run.gen();
|
||||
run.onCallResult(gen, 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
stepRun(run, ok, 2.5); // poll in flight
|
||||
EXPECT_TRUE(run.abort());
|
||||
EXPECT_TRUE(run.end() == RunEnd::Aborted && run.batches()[0].state == OpState::Lost);
|
||||
EXPECT_TRUE(run.addresses()[0].outcome == AddrOutcome::Uncertain);
|
||||
EXPECT_EQ(run.onCallResult(gen, 0, MergeCallResult{"o9", 5, 5, ""}, K::None, 0.1, 3.0), std::string("o9"));
|
||||
run.onPollResult(gen, true, {opRep("o1", "success")}, 3.0);
|
||||
EXPECT_TRUE(run.batches()[0].state == OpState::Lost);
|
||||
EXPECT_TRUE(run.next(ok, 4.0).action == RunAction::None);
|
||||
EXPECT_FALSE(run.abort());
|
||||
}
|
||||
{ // Watchdogs: a call or poll outstanding 30 min is presumed lost; an op not terminal after an hour too.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 200, 100.0}, {"zB", 10, 5.0}}, RunConfig{}, 0.0);
|
||||
stepRun(run, ok, 0.0);
|
||||
const auto gen = run.gen();
|
||||
stepRun(run, ok, 1801.0);
|
||||
EXPECT_TRUE(run.addresses()[0].done && run.addresses()[0].outcome == AddrOutcome::Uncertain);
|
||||
EXPECT_EQ(run.gen(), gen + 1);
|
||||
EXPECT_EQ(run.onCallResult(gen, 0, MergeCallResult{"late", 5, 5, ""}, K::None, 0.1, 1802.0), std::string("late"));
|
||||
ConsolidationRun lost;
|
||||
lost.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(lost, ok, 0.0);
|
||||
lost.onCallResult(lost.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
stepRun(lost, ok, 0.2);
|
||||
lost.onPollResult(lost.gen(), true, {}, 0.3); // missing once
|
||||
EXPECT_TRUE(lost.batches()[0].state == OpState::Unknown);
|
||||
stepRun(lost, ok, 3.5);
|
||||
lost.onPollResult(lost.gen(), true, {}, 3.6); // twice: Lost
|
||||
EXPECT_TRUE(lost.batches()[0].state == OpState::Lost);
|
||||
ConsolidationRun hour;
|
||||
hour.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(hour, ok, 0.0);
|
||||
hour.onCallResult(hour.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
for (double t = 0.5; t < 3610.0; t += 30.0) {
|
||||
const auto s2 = stepRun(hour, ok, t);
|
||||
if (s2.action == RunAction::Poll) hour.onPollResult(hour.gen(), false, {}, t + 0.1);
|
||||
}
|
||||
EXPECT_TRUE(hour.batches()[0].state == OpState::Lost);
|
||||
}
|
||||
{ // The op backstop counts only time in which a poll could have gone out (not while the node was gone or
|
||||
// busy), with a 4 h cap whatever happened.
|
||||
ConsolidationRun gated;
|
||||
gated.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(gated, ok, 0.0);
|
||||
gated.onCallResult(gated.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
RunEnv gone = ok;
|
||||
gone.nodeUsable = false;
|
||||
for (double t = 1.0; t < 7200.0; t += 30.0) stepRun(gated, gone, t);
|
||||
EXPECT_FALSE(opStateTerminal(gated.batches()[0].state));
|
||||
EXPECT_TRUE(stepRun(gated, ok, 7201.0).action == RunAction::Poll);
|
||||
gated.onPollResult(gated.gen(), true, {opRep("o1", "success")}, 7201.1);
|
||||
EXPECT_TRUE(gated.batches()[0].state == OpState::Success);
|
||||
ConsolidationRun capped;
|
||||
capped.start({{"zA", 200, 100.0}}, RunConfig{}, 0.0);
|
||||
stepRun(capped, ok, 0.0);
|
||||
capped.onCallResult(capped.gen(), 0, MergeCallResult{"o1", 50, 150, ""}, K::None, 0.1, 0.1);
|
||||
RunEnv busy = ok;
|
||||
busy.busy = true;
|
||||
for (double t = 1.0; t < 4 * 3600.0 + 60.0; t += 60.0) stepRun(capped, busy, t);
|
||||
EXPECT_TRUE(capped.batches()[0].state == OpState::Lost);
|
||||
}
|
||||
{ // The estimate is capped by maxBatches.
|
||||
ConsolidationRun run;
|
||||
run.start({{"zA", 10000, 5000.0}}, RunConfig{}, 0.0);
|
||||
EXPECT_EQ(run.progress().batchesEstimated, 20);
|
||||
}
|
||||
}
|
||||
|
||||
// A whole run against a deterministic fake daemon: serial ops, merge calls that cost 6 s, and windows of lock,
|
||||
// catch-up and another wallet scan.
|
||||
void testConsolidationRunSimulation()
|
||||
{
|
||||
using namespace dragonx::services;
|
||||
struct FakeOp { std::string id; std::string addr; int notes; double startAt = -1, finishAt = -1; };
|
||||
std::map<std::string, int> unlocked{{"A", 240}, {"B", 60}, {"C", 3}, {"D", 1}};
|
||||
std::vector<FakeOp> ops;
|
||||
ConsolidationRun run;
|
||||
std::vector<NoteGroup> groups;
|
||||
for (const auto& [a, n] : unlocked) groups.push_back({a, n, 0.5 * n});
|
||||
EXPECT_TRUE(run.start(groups, RunConfig{}, 0.0));
|
||||
EXPECT_EQ(run.addresses().size(), static_cast<size_t>(3)); // D (one note) is left out
|
||||
int expected = 0;
|
||||
for (const auto& a : run.addresses()) expected += estimateBatches(a.group.notes, 50);
|
||||
// In-flight RPC model.
|
||||
enum class Rpc { None, Call, Poll } rpc = Rpc::None;
|
||||
double rpcDoneAt = 0.0, lastCallEnd = -1e9;
|
||||
int rpcAddr = -1;
|
||||
bool overlap = false, gapOk = true, windowOk = true, opsOk = true;
|
||||
auto inWindow = [](double t, double a, double b) { return t >= a && t < b; };
|
||||
auto settle = [&](double t) { // the daemon's serial op queue
|
||||
double prevFinish = 0.0;
|
||||
for (auto& op : ops) {
|
||||
if (op.startAt < 0) op.startAt = std::max(t, prevFinish); // set when queued below
|
||||
prevFinish = op.finishAt;
|
||||
}
|
||||
};
|
||||
(void)settle;
|
||||
auto status = [&](const FakeOp& op, double t) {
|
||||
if (t < op.startAt) return std::string("queued");
|
||||
if (t < op.finishAt) return std::string("executing");
|
||||
return std::string("success");
|
||||
};
|
||||
double t = 0.0;
|
||||
for (; t < 5000.0 && run.hasRun() && run.end() == RunEnd::None; t += 0.5) {
|
||||
// Replies land.
|
||||
if (rpc != Rpc::None && t >= rpcDoneAt) {
|
||||
if (rpc == Rpc::Call) {
|
||||
const std::string addr = run.addresses()[rpcAddr].group.address;
|
||||
MergeCallResult r;
|
||||
const int avail = unlocked[addr];
|
||||
if (avail < 2) {
|
||||
r.error = "Could not find any funds to merge.";
|
||||
run.onCallResult(run.gen(), rpcAddr, r, dragonx::rpc::RpcFailureKind::AppError, 6.0, t);
|
||||
} else {
|
||||
const int merging = std::min(50, avail);
|
||||
unlocked[addr] -= merging; // the op locks its notes before the call returns
|
||||
if (unlocked[addr] < 0) opsOk = false;
|
||||
FakeOp op;
|
||||
op.id = "op" + std::to_string(ops.size());
|
||||
op.addr = addr;
|
||||
op.notes = merging;
|
||||
const double prev = ops.empty() ? 0.0 : ops.back().finishAt;
|
||||
op.startAt = std::max(t, prev);
|
||||
op.finishAt = op.startAt + 8.0;
|
||||
ops.push_back(op);
|
||||
r.opid = op.id;
|
||||
r.merging = merging;
|
||||
r.remaining = unlocked[addr];
|
||||
run.onCallResult(run.gen(), rpcAddr, r, dragonx::rpc::RpcFailureKind::None, 6.0, t);
|
||||
}
|
||||
lastCallEnd = t;
|
||||
} else {
|
||||
std::vector<OpReport> reps;
|
||||
for (const auto& id : run.outstandingOpids()) {
|
||||
for (const auto& op : ops) if (op.id == id) reps.push_back(opRep(id, status(op, t)));
|
||||
}
|
||||
run.onPollResult(run.gen(), true, reps, t);
|
||||
}
|
||||
rpc = Rpc::None;
|
||||
}
|
||||
int unfinished = 0;
|
||||
for (const auto& op : ops) if (status(op, t) != "success") ++unfinished;
|
||||
if (unfinished > 2) opsOk = false;
|
||||
RunEnv env = okRunEnv();
|
||||
env.locked = inWindow(t, 100.0, 160.0);
|
||||
env.callsAllowed = !inWindow(t, 300.0, 330.0);
|
||||
env.scanBusy = inWindow(t, 400.0, 410.0);
|
||||
const RunStep st = run.next(env, t);
|
||||
if (st.action == RunAction::Call) {
|
||||
if (rpc != Rpc::None) overlap = true;
|
||||
if (t - lastCallEnd < 6.0 - 1e-9) gapOk = false;
|
||||
if (inWindow(t, 100.0, 160.0) || inWindow(t, 300.0, 330.0) || inWindow(t, 400.0, 410.0)) windowOk = false;
|
||||
run.markCallPosted(st.addrIdx, t);
|
||||
rpc = Rpc::Call;
|
||||
rpcAddr = st.addrIdx;
|
||||
rpcDoneAt = t + 6.0;
|
||||
} else if (st.action == RunAction::Poll) {
|
||||
if (rpc != Rpc::None) overlap = true;
|
||||
run.markPollPosted(t);
|
||||
rpc = Rpc::Poll;
|
||||
rpcDoneAt = t + 0.1;
|
||||
}
|
||||
}
|
||||
EXPECT_TRUE(run.end() == RunEnd::Completed);
|
||||
EXPECT_FALSE(overlap);
|
||||
EXPECT_TRUE(gapOk);
|
||||
EXPECT_TRUE(windowOk);
|
||||
EXPECT_TRUE(opsOk);
|
||||
const auto sum = run.summary();
|
||||
EXPECT_EQ(sum.batchesSubmitted, expected);
|
||||
EXPECT_EQ(expected, 8);
|
||||
for (const auto& a : run.addresses()) EXPECT_TRUE(a.run.remaining < 2);
|
||||
EXPECT_TRUE(runVerdict(sum) == RunVerdict::Clean);
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
testConnectionConfig();
|
||||
@@ -11672,6 +12283,11 @@ int main()
|
||||
testDebugLogTailRules();
|
||||
testNoteConsolidation();
|
||||
testScopeScanAndLockedNotes();
|
||||
testMergeCallResult();
|
||||
testOpReports();
|
||||
testMergeFailureClassification();
|
||||
testConsolidationRun();
|
||||
testConsolidationRunSimulation();
|
||||
testStopDeliveryPolicy();
|
||||
testProcStatParse();
|
||||
testStopProgressTracker();
|
||||
|
||||
Reference in New Issue
Block a user