feat(daemon): deliver stop reliably and never kill the node unasked
Closing the wallet sent "stop" at most twice, gated on a getinfo probe that needs the daemon's cs_main, then killed the node after 20 s (Windows) or 30 s (POSIX) whether or not stop had arrived. A node busy with an hour-long witness rebuild therefore never got stop, was killed mid- rebuild, and replayed everything since its last flush on every start. Delivery (src/daemon/stop_delivery.h, RPCClient::open/trySendStop): - stop goes over a dedicated client with no probe, so it never waits on rpc_'s lock or a cs_main-bound getinfo - every attempt is classified, and StopDeliveryPolicy decides the next: Accepted / 503 = delivered; Queued (reply timeout) is resent at most every 30 s; QueueFull / not listening are retried every 1.5 s; a refusal after we were heard means the node is shutting down; a 401 on our own node tries the next credential source - POSIX: a group SIGTERM goes out as soon as an attempt fails, which bypasses the HTTP queue entirely Waiting (DaemonController::executeDaemonStop, pure, virtual-clock tested): - never kills on its own. Quitting waits as long as the node needs; the shutdown screen offers "Close wallet — leave node running" after 3 s, and "Kill node (unsafe)" (confirmed) only once stop was delivered and the node showed no CPU/I-O progress for ~10 min outside a rebuild - other callers (restart, rescan, reinstall, switch, ...) wait up to 120 s, let go once the app starts quitting, and keep watching a node that is still running; wallet switch refuses to go ahead then - EmbeddedDaemon gains stop-session primitives (beginStop, pollOwnedProcess, signalGracefulStop, forceKillOwnedProcess, ownedResourceUsage over the Linux process group / Windows handle, endStop) and only one stop session runs at a time - the Force Quit button and its stall heuristic are gone; "Leave running" aborts a pending stop attempt and closes at once Verified against the fake daemon with the real GUI: closing mid-rebuild now delivers stop (or SIGTERM 2 ms after a queue-full refusal) and the node exits cleanly after its rebuild, where it was SIGKILLed before. The shutdown screen, its "Leave running" button and the idle Kill offer (after ~11 min) were checked by screenshot. New strings are English- only for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,155 @@
|
||||
#pragma once
|
||||
|
||||
// Delivering "stop" to dragonxd: what one attempt achieved, and when to send another. Pure and
|
||||
// header-only so the test target can drive it without the RPC client.
|
||||
//
|
||||
// Why retrying matters: dragonxd's HTTP server has 8 workers and a 16-deep queue. While a long
|
||||
// operation holds cs_main, the GUI's own abandoned polls can fill every slot, and each new request is
|
||||
// refused at once with "Work queue depth exceeded" until a slot frees. The old code made one or two
|
||||
// attempts, gated on a getinfo probe that needs cs_main, so stop was never delivered and the node was
|
||||
// then killed.
|
||||
|
||||
#include "rpc/rpc_failure.h"
|
||||
|
||||
namespace dragonx {
|
||||
namespace daemon {
|
||||
|
||||
// What the user picked on the shutdown screen while the node was stopping.
|
||||
enum class StopUserChoice { None, Detach, Kill };
|
||||
|
||||
// How a stop session ended. The wallet never kills the node unless the user chose Kill.
|
||||
enum class StopOutcome {
|
||||
Exited, // the process is gone
|
||||
StillRunning, // we stopped waiting (time budget, app shutdown, cancellation); never killed
|
||||
Detached, // the user chose to leave it running
|
||||
Killed, // the user chose Kill
|
||||
};
|
||||
|
||||
inline const char* stopOutcomeName(StopOutcome o)
|
||||
{
|
||||
switch (o) {
|
||||
case StopOutcome::Exited: return "Exited";
|
||||
case StopOutcome::StillRunning: return "StillRunning";
|
||||
case StopOutcome::Detached: return "Detached";
|
||||
case StopOutcome::Killed: return "Killed";
|
||||
}
|
||||
return "Exited";
|
||||
}
|
||||
|
||||
enum class StopSendResult {
|
||||
Accepted, // HTTP 200: the daemon ran stop
|
||||
Queued, // reply timeout: it reached the daemon and waits for a worker; it will run
|
||||
QueueFull, // "Work queue depth exceeded": refused before queueing (holds no slot)
|
||||
NotListening, // refused / connect timeout: RPC isn't bound (not up yet, or already unbound)
|
||||
ShuttingDown, // 503 / "Unhandled request": the daemon is already shutting down
|
||||
Auth, // 401: our credentials are wrong or stale
|
||||
Other,
|
||||
};
|
||||
|
||||
inline const char* stopSendResultName(StopSendResult r)
|
||||
{
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted: return "Accepted";
|
||||
case StopSendResult::Queued: return "Queued";
|
||||
case StopSendResult::QueueFull: return "QueueFull";
|
||||
case StopSendResult::NotListening: return "NotListening";
|
||||
case StopSendResult::ShuttingDown: return "ShuttingDown";
|
||||
case StopSendResult::Auth: return "Auth";
|
||||
case StopSendResult::Other: return "Other";
|
||||
}
|
||||
return "Other";
|
||||
}
|
||||
|
||||
// What a stop call's failure kind (RpcFailureKind::None = HTTP 200) means for delivery.
|
||||
inline StopSendResult stopSendResultFromKind(rpc::RpcFailureKind kind)
|
||||
{
|
||||
using K = rpc::RpcFailureKind;
|
||||
switch (kind) {
|
||||
case K::None: return StopSendResult::Accepted;
|
||||
case K::ReplyTimeout: return StopSendResult::Queued;
|
||||
case K::QueueFull: return StopSendResult::QueueFull;
|
||||
case K::NotListening:
|
||||
case K::ConnectTimeout: return StopSendResult::NotListening;
|
||||
case K::ShuttingDown:
|
||||
case K::Unhandled: return StopSendResult::ShuttingDown;
|
||||
case K::Auth: return StopSendResult::Auth;
|
||||
default: return StopSendResult::Other;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When to (re)send stop. Times are seconds on the caller's clock.
|
||||
*
|
||||
* - Accepted / ShuttingDown: delivered, never resend.
|
||||
* - Queued: the daemon has it. Resend at most every 30s: each resend takes another queue slot, and
|
||||
* concurrent stops race on the miner threads (stop's GenerateBitcoins(false)).
|
||||
* - QueueFull / NotListening / Other: not delivered, cheap to retry: again after 1.5s.
|
||||
* - NotListening AFTER the daemon had heard us means it unbound its listener to shut down: delivered.
|
||||
* - Auth: give up for a node we don't own. For our own node the caller may switch credentials and
|
||||
* retry at once (retryNow), or give up (giveUp).
|
||||
*/
|
||||
class StopDeliveryPolicy {
|
||||
public:
|
||||
static constexpr double kRetrySec = 1.5;
|
||||
static constexpr double kQueuedResendSec = 30.0;
|
||||
|
||||
explicit StopDeliveryPolicy(bool owned) : owned_(owned) {}
|
||||
|
||||
bool shouldSend(double now) const { return !finished_ && now >= nextSendAt_; }
|
||||
|
||||
// Records one attempt; returns it as interpreted (a refusal after we were heard reads as ShuttingDown).
|
||||
StopSendResult record(StopSendResult r, double now)
|
||||
{
|
||||
++attempts_;
|
||||
if (r == StopSendResult::NotListening && heard_) r = StopSendResult::ShuttingDown;
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted:
|
||||
heard_ = true;
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::ShuttingDown:
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::Queued:
|
||||
heard_ = true;
|
||||
nextSendAt_ = now + kQueuedResendSec;
|
||||
break;
|
||||
case StopSendResult::Auth:
|
||||
authRejected_ = true;
|
||||
if (!owned_) finished_ = true;
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
case StopSendResult::QueueFull:
|
||||
case StopSendResult::NotListening:
|
||||
case StopSendResult::Other:
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
}
|
||||
last_ = r;
|
||||
return r;
|
||||
}
|
||||
|
||||
void markDelivered() { delivered_ = finished_ = true; } // also for a graceful signal (POSIX SIGTERM)
|
||||
void retryNow(double now) { nextSendAt_ = now; }
|
||||
void giveUp() { finished_ = true; }
|
||||
|
||||
bool delivered() const { return delivered_; } // the daemon is stopping (or was signalled)
|
||||
bool heard() const { return heard_; } // it has (or ran) our request
|
||||
bool finished() const { return finished_; } // no more attempts
|
||||
bool authRejected() const { return authRejected_; }
|
||||
int attempts() const { return attempts_; }
|
||||
StopSendResult last() const { return last_; }
|
||||
|
||||
private:
|
||||
bool owned_ = true;
|
||||
bool delivered_ = false;
|
||||
bool heard_ = false;
|
||||
bool finished_ = false;
|
||||
bool authRejected_ = false;
|
||||
int attempts_ = 0;
|
||||
double nextSendAt_ = 0.0;
|
||||
StopSendResult last_ = StopSendResult::Other;
|
||||
};
|
||||
|
||||
} // namespace daemon
|
||||
} // namespace dragonx
|
||||
Reference in New Issue
Block a user