feat(rpc): stop flooding a busy node's RPC queue; fix the reconnect storm
dragonxd serves RPC with 8 workers and a 16-deep queue. While a witness rebuild holds cs_main, every request needing it waits, and a request the client gave up on keeps its slot. The GUI kept polling and re-probing until all 24 slots were its own abandoned requests (even stop was then refused), treated the busy node as lost (tearing down the wallet state), and retried connect at frame rate on some pages. Busy gate (src/rpc/rpc_busy_gate.h), enforced in RPCClient::performCall for the shared rpc_ / fast_rpc_: - the first reply timeout or queue-full refusal marks the node busy; from then on nothing needing cs_main is sent (calls fail at once with the new RpcFailureKind::Deferred). stop, help and getrescaninfo are lock-free in dragonxd and always go through - one canary getinfo with no client timeout waits on its own client and thread; its answer clears the gate and triggers an immediate refresh (queue-full: retried after 2..30 s; no answer: back to normal handling) - the timer dispatch skips Core/Tx/Addresses/Peers/chat/mining/opid and connect attempts while busy - the 3-strike teardown counts only failures that put the connection in doubt, never busy ones; no balance scans follow a busy chain-info call Reconnect storm: retries were armed with setTimer(Core, kCoreDefault - 1), i.e. "4 s elapsed", which against Overview's 2 s interval meant every frame. RefreshScheduler::deferFor() is an absolute hold, and ConnectRetryBackoff picks the delay by failure kind (not listening 1/2/5 s, queue full 2..30 s, busy 30 s, else 4 s). Measured against the fake daemon: abandoned requests during a rebuild went from ~24 (every slot) to at most 2, with no teardown and recovery when the rebuild ends; a saturated queue sees about one canary per back-off step instead of ~190 requests/s. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+159
-1
@@ -24,6 +24,7 @@
|
||||
#include "data/wallet_state.h"
|
||||
#include "rpc/connection.h"
|
||||
#include "rpc/rpc_failure.h"
|
||||
#include "rpc/rpc_busy_gate.h"
|
||||
#include "resources/embedded_resources.h"
|
||||
#include "services/network_refresh_service.h"
|
||||
#include "services/refresh_scheduler.h"
|
||||
@@ -1077,6 +1078,58 @@ void testSpendableFiltering()
|
||||
EXPECT_EQ(includeZero.size(), static_cast<size_t>(3));
|
||||
}
|
||||
|
||||
// S8: deferFor() is an absolute hold, independent of the page interval (the retry-storm fix).
|
||||
void testRefreshSchedulerDeferFor()
|
||||
{
|
||||
using dragonx::services::RefreshScheduler;
|
||||
using Timer = RefreshScheduler::Timer;
|
||||
{ // The old bug, pinned: "setTimer(kCoreDefault - 1)" meant 4s ELAPSED, and Overview's interval is 2s,
|
||||
// so the retry was due at once (every frame).
|
||||
RefreshScheduler s;
|
||||
s.applyPage(dragonx::ui::NavPage::Overview);
|
||||
s.setTimer(Timer::Core, RefreshScheduler::kCoreDefault - 1.0f);
|
||||
EXPECT_TRUE(s.isDue(Timer::Core));
|
||||
}
|
||||
{ // Overview + deferFor(4): not due at 3.9, due at 4.0.
|
||||
RefreshScheduler s;
|
||||
s.applyPage(dragonx::ui::NavPage::Overview);
|
||||
s.deferFor(Timer::Core, 4.0f);
|
||||
s.tick(3.9f);
|
||||
EXPECT_FALSE(s.isDue(Timer::Core));
|
||||
s.tick(0.1f);
|
||||
EXPECT_TRUE(s.isDue(Timer::Core));
|
||||
}
|
||||
{ // A page change mid-hold doesn't move it.
|
||||
RefreshScheduler s;
|
||||
s.applyPage(dragonx::ui::NavPage::Overview);
|
||||
s.deferFor(Timer::Core, 4.0f);
|
||||
s.tick(1.0f);
|
||||
s.applyPage(dragonx::ui::NavPage::Console); // 10s interval
|
||||
s.tick(2.9f);
|
||||
EXPECT_FALSE(s.isDue(Timer::Core));
|
||||
s.tick(0.1f);
|
||||
EXPECT_TRUE(s.isDue(Timer::Core));
|
||||
}
|
||||
{ // Shorter than the interval: due at 1.0 on Console, not at 10.
|
||||
RefreshScheduler s;
|
||||
s.applyPage(dragonx::ui::NavPage::Console);
|
||||
s.deferFor(Timer::Core, 1.0f);
|
||||
s.tick(1.0f);
|
||||
EXPECT_TRUE(s.consumeDue(Timer::Core));
|
||||
// consumeDue ended the hold: back to the 10s interval.
|
||||
s.tick(5.0f);
|
||||
EXPECT_FALSE(s.isDue(Timer::Core));
|
||||
}
|
||||
{ // A user action (markDue / markImmediateRefresh) overrides a backoff.
|
||||
RefreshScheduler s;
|
||||
s.applyPage(dragonx::ui::NavPage::Overview);
|
||||
s.deferFor(Timer::Core, 30.0f);
|
||||
s.tick(1.0f);
|
||||
s.markImmediateRefresh();
|
||||
EXPECT_TRUE(s.isDue(Timer::Core));
|
||||
}
|
||||
}
|
||||
|
||||
void testRefreshScheduler()
|
||||
{
|
||||
using dragonx::services::RefreshScheduler;
|
||||
@@ -1612,7 +1665,10 @@ void testNetworkRefreshRpcCollectors()
|
||||
EXPECT_FALSE(busyCore.blockchainOk);
|
||||
EXPECT_FALSE(busyCore.balanceOk);
|
||||
EXPECT_TRUE(busyCore.blockchainErrKind == RpcFailureKind::QueueFull);
|
||||
EXPECT_TRUE(busyCore.balanceErrKind == RpcFailureKind::ReplyTimeout);
|
||||
// S6: after the chain call found the node busy, no balance scan is sent at all (each would only
|
||||
// queue behind the same cs_main and take another slot).
|
||||
EXPECT_TRUE(busyRpc.methodNames() == std::vector<std::string>({"getblockchaininfo"}));
|
||||
EXPECT_TRUE(busyCore.balanceErrKind == RpcFailureKind::None);
|
||||
}
|
||||
|
||||
MockRefreshRpc peerRpc;
|
||||
@@ -4167,6 +4223,104 @@ void testOutputSinceAcrossTrim()
|
||||
EXPECT_TRUE(got.empty());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
// S6: the busy gate (don't flood a busy node's RPC queue) and the connect backoff.
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
|
||||
void testRpcBusyGate()
|
||||
{
|
||||
using dragonx::rpc::RpcBusyGate;
|
||||
using K = dragonx::rpc::RpcFailureKind;
|
||||
RpcBusyGate g;
|
||||
EXPECT_TRUE(g.admit("getinfo"));
|
||||
EXPECT_FALSE(g.busy());
|
||||
|
||||
// The first reply timeout on a cs_main call flips to busy: nothing needing cs_main goes out.
|
||||
g.record("getblockchaininfo", K::ReplyTimeout);
|
||||
EXPECT_TRUE(g.busy());
|
||||
EXPECT_EQ(g.abandoned(), 1);
|
||||
EXPECT_EQ(g.busyPeriods(), 1);
|
||||
EXPECT_FALSE(g.admit("getinfo"));
|
||||
EXPECT_FALSE(g.admit("z_gettotalbalance"));
|
||||
EXPECT_TRUE(g.admit("stop")); // lock-free: always
|
||||
EXPECT_TRUE(g.admit("help"));
|
||||
EXPECT_TRUE(g.admit("getrescaninfo"));
|
||||
|
||||
// Lock-free calls say nothing about cs_main; "nothing listening" says nothing about busyness.
|
||||
g.record("help", K::ReplyTimeout);
|
||||
g.record("getinfo", K::NotListening);
|
||||
EXPECT_EQ(g.abandoned(), 1);
|
||||
EXPECT_TRUE(g.busy());
|
||||
g.record("getinfo", K::ReplyTimeout); // a second timeout in the same busy period
|
||||
EXPECT_EQ(g.busyPeriods(), 1); // ...is not a new period (one canary per period)
|
||||
|
||||
// Any answer clears it, even an application error.
|
||||
g.record("z_sendmany", K::AppError);
|
||||
EXPECT_FALSE(g.busy());
|
||||
EXPECT_EQ(g.abandoned(), 0);
|
||||
|
||||
// Queue full is busy too, but the refused request holds no slot.
|
||||
g.record("getinfo", K::QueueFull);
|
||||
EXPECT_TRUE(g.busy());
|
||||
EXPECT_EQ(g.abandoned(), 0);
|
||||
EXPECT_EQ(g.busyPeriods(), 2);
|
||||
g.clear();
|
||||
EXPECT_FALSE(g.busy());
|
||||
|
||||
// An hour of polling on two lanes against a node stuck on cs_main: only what was already in flight
|
||||
// when it turned busy gets abandoned; everything after is held back. (Was ~24: every slot.)
|
||||
RpcBusyGate hour;
|
||||
int sent = 0;
|
||||
for (int second = 0; second < 3600; ++second) {
|
||||
for (int lane = 0; lane < 2; ++lane) {
|
||||
if (!hour.admit("getblockchaininfo")) continue;
|
||||
++sent;
|
||||
if (second > 0 || lane == 1) hour.record("getblockchaininfo", K::ReplyTimeout);
|
||||
}
|
||||
}
|
||||
EXPECT_TRUE(sent <= 2);
|
||||
EXPECT_TRUE(hour.abandoned() <= 2);
|
||||
}
|
||||
|
||||
void testRpcKindClasses()
|
||||
{
|
||||
using namespace dragonx::rpc;
|
||||
using K = RpcFailureKind;
|
||||
for (K k : {K::ReplyTimeout, K::QueueFull, K::Deferred}) {
|
||||
EXPECT_TRUE(rpcKindMeansBusy(k));
|
||||
EXPECT_FALSE(rpcKindMeansLost(k)); // a busy node is still there: never a reconnect
|
||||
}
|
||||
for (K k : {K::NotListening, K::ConnectTimeout, K::Auth, K::ShuttingDown, K::Unhandled, K::Other}) {
|
||||
EXPECT_FALSE(rpcKindMeansBusy(k));
|
||||
EXPECT_TRUE(rpcKindMeansLost(k));
|
||||
}
|
||||
for (K k : {K::None, K::Warmup, K::AppError, K::Aborted}) {
|
||||
EXPECT_FALSE(rpcKindMeansBusy(k));
|
||||
EXPECT_FALSE(rpcKindMeansLost(k));
|
||||
}
|
||||
}
|
||||
|
||||
void testConnectRetryBackoff()
|
||||
{
|
||||
using dragonx::rpc::ConnectRetryBackoff;
|
||||
using K = dragonx::rpc::RpcFailureKind;
|
||||
ConnectRetryBackoff b;
|
||||
EXPECT_NEAR(b.next(K::NotListening), 1.0, 1e-6);
|
||||
EXPECT_NEAR(b.next(K::NotListening), 2.0, 1e-6);
|
||||
EXPECT_NEAR(b.next(K::ConnectTimeout), 1.0, 1e-6); // a different kind starts over
|
||||
b.reset();
|
||||
const double qf[] = {2, 4, 8, 16, 30, 30};
|
||||
for (double want : qf) EXPECT_NEAR(b.next(K::QueueFull), want, 1e-6);
|
||||
EXPECT_NEAR(b.next(K::ReplyTimeout), 30.0, 1e-6); // the canary owns recovery
|
||||
EXPECT_NEAR(b.next(K::Other), 4.0, 1e-6);
|
||||
// A minute of queue-full refusals: a handful of attempts, not thousands (was ~3,000/min).
|
||||
b.reset();
|
||||
double t = 0;
|
||||
int attempts = 0;
|
||||
while (t < 60.0) { t += b.next(K::QueueFull); ++attempts; }
|
||||
EXPECT_TRUE(attempts <= 6);
|
||||
}
|
||||
|
||||
// S3: on Linux a node in its final shutdown phase reads comm "hush-shutoff"; its executable still says
|
||||
// dragonxd, so it must still count as running (it holds the datadir lock until it exits).
|
||||
void testDaemonProcessIdentity()
|
||||
@@ -8737,6 +8891,10 @@ int main()
|
||||
testAmountFormatting();
|
||||
testSpendableFiltering();
|
||||
testRefreshScheduler();
|
||||
testRefreshSchedulerDeferFor();
|
||||
testRpcBusyGate();
|
||||
testRpcKindClasses();
|
||||
testConnectRetryBackoff();
|
||||
testNetworkRefreshService();
|
||||
testNetworkRefreshSnapshotHelpers();
|
||||
testParseDaemonRescanOutput();
|
||||
|
||||
Reference in New Issue
Block a user