feat(daemon): deliver stop reliably and never kill the node unasked
Closing the wallet sent "stop" at most twice, gated on a getinfo probe that needs the daemon's cs_main, then killed the node after 20 s (Windows) or 30 s (POSIX) whether or not stop had arrived. A node busy with an hour-long witness rebuild therefore never got stop, was killed mid- rebuild, and replayed everything since its last flush on every start. Delivery (src/daemon/stop_delivery.h, RPCClient::open/trySendStop): - stop goes over a dedicated client with no probe, so it never waits on rpc_'s lock or a cs_main-bound getinfo - every attempt is classified, and StopDeliveryPolicy decides the next: Accepted / 503 = delivered; Queued (reply timeout) is resent at most every 30 s; QueueFull / not listening are retried every 1.5 s; a refusal after we were heard means the node is shutting down; a 401 on our own node tries the next credential source - POSIX: a group SIGTERM goes out as soon as an attempt fails, which bypasses the HTTP queue entirely Waiting (DaemonController::executeDaemonStop, pure, virtual-clock tested): - never kills on its own. Quitting waits as long as the node needs; the shutdown screen offers "Close wallet — leave node running" after 3 s, and "Kill node (unsafe)" (confirmed) only once stop was delivered and the node showed no CPU/I-O progress for ~10 min outside a rebuild - other callers (restart, rescan, reinstall, switch, ...) wait up to 120 s, let go once the app starts quitting, and keep watching a node that is still running; wallet switch refuses to go ahead then - EmbeddedDaemon gains stop-session primitives (beginStop, pollOwnedProcess, signalGracefulStop, forceKillOwnedProcess, ownedResourceUsage over the Linux process group / Windows handle, endStop) and only one stop session runs at a time - the Force Quit button and its stall heuristic are gone; "Leave running" aborts a pending stop attempt and closes at once Verified against the fake daemon with the real GUI: closing mid-rebuild now delivers stop (or SIGTERM 2 ms after a queue-full refusal) and the node exits cleanly after its rebuild, where it was SIGKILLed before. The shutdown screen, its "Leave running" button and the idle Kill offer (after ~11 min) were checked by screenshot. New strings are English- only for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "embedded_daemon.h"
|
||||
#include "stop_delivery.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -234,6 +236,215 @@ public:
|
||||
ShutdownDecision shutdownDecision(bool keepDaemonRunning,
|
||||
bool stopExternalDaemon) const;
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Stopping the node: deliver "stop" until it lands, then wait for the process to exit. Never kill it
|
||||
// on our own: a node that is busy (a witness rebuild can hold it for an hour) or flushing would lose
|
||||
// everything since its last save and replay it on the next start. Only an explicit user "Kill",
|
||||
// offered once the node has gone idle, ends it by force; "Leave running" lets the wallet close while
|
||||
// the node finishes on its own.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
using StopUserChoice = daemon::StopUserChoice;
|
||||
using StopOutcome = daemon::StopOutcome;
|
||||
|
||||
struct StopPolicy {
|
||||
bool owned = true; // we launched it: watch the process, signal it, offer Kill
|
||||
bool gracefulSignal = false; // POSIX: a group SIGTERM is a stop channel (dragonxd treats it as stop)
|
||||
double maxWaitSec = 0.0; // stop waiting (never kill) after this long; 0 = no limit
|
||||
bool userChoices = false; // the shutdown screen offers "Leave running" / "Kill"
|
||||
bool abandonOnShutdown = false; // stop waiting once the app starts shutting down (background callers)
|
||||
double idleOfferSec = 600.0; // delivered + no CPU/I-O progress for this long: offer Kill
|
||||
};
|
||||
|
||||
// What the stop loop reports for display.
|
||||
struct StopStatus {
|
||||
bool delivered = false; // stop accepted (or the node is shutting down, or was signalled)
|
||||
bool heard = false; // the node has the request (possibly still queued)
|
||||
bool signalled = false; // a graceful signal (POSIX SIGTERM) went out
|
||||
double elapsedSec = 0.0;
|
||||
double cpuPercent = -1.0; // < 0: unknown
|
||||
double idleSec = 0.0;
|
||||
bool killOffered = false;
|
||||
bool rebuildActive = false;
|
||||
StopSendResult lastSend = StopSendResult::Other;
|
||||
int attempts = 0;
|
||||
};
|
||||
|
||||
class StopRuntime {
|
||||
public:
|
||||
virtual ~StopRuntime() = default;
|
||||
virtual double nowSec() = 0;
|
||||
virtual bool processAlive() = 0;
|
||||
virtual StopSendResult trySendStop() = 0;
|
||||
virtual bool nextCredentials() = 0; // switch to the next stop credentials; false when none left
|
||||
virtual bool sendGracefulSignal() = 0; // POSIX group SIGTERM; false where there is none (Windows)
|
||||
virtual double cpuSeconds() = 0; // cumulative CPU of the node; < 0 if unknown
|
||||
virtual double ioBytes() = 0; // cumulative I/O bytes of the node; < 0 if unknown
|
||||
virtual bool rebuildActive() = 0; // a witness rebuild is running: never offer Kill
|
||||
virtual StopUserChoice userChoice() = 0;
|
||||
virtual void offerForceKill(bool offered) = 0;
|
||||
virtual void forceKill() = 0;
|
||||
virtual void setStatus(const StopStatus& status) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* "Making progress" = >= 0.05 cores of CPU or >= 1 MB of I/O over the last >= 60s. A flush that
|
||||
* fsyncs a big wallet is mostly I/O at low CPU; a rebuild is mostly CPU. Idle time counts from the
|
||||
* first sample. With no readings at all (unknown), it never reports idle, so Kill is never offered.
|
||||
*/
|
||||
class StopProgressTracker {
|
||||
public:
|
||||
static constexpr double kWindowSec = 60.0;
|
||||
static constexpr double kMinCores = 0.05;
|
||||
static constexpr double kMinIoBytes = 1024.0 * 1024.0;
|
||||
|
||||
void add(double t, double cpuSec, double ioBytes)
|
||||
{
|
||||
if (cpuSec >= 0 || ioBytes >= 0) measurable_ = true;
|
||||
if (samples_.empty()) lastProgressAt_ = t;
|
||||
samples_.push_back({t, cpuSec, ioBytes});
|
||||
// Keep the newest sample that is at least a full window old as the comparison base.
|
||||
while (samples_.size() > 2 && samples_[1].t <= t - kWindowSec) samples_.pop_front();
|
||||
const Sample& base = samples_.front();
|
||||
const double span = t - base.t;
|
||||
if (span >= kWindowSec) {
|
||||
const bool cpuMoved = cpuSec >= 0 && base.cpu >= 0 && (cpuSec - base.cpu) / span >= kMinCores;
|
||||
const bool ioMoved = ioBytes >= 0 && base.io >= 0 && (ioBytes - base.io) >= kMinIoBytes;
|
||||
if (cpuMoved || ioMoved) lastProgressAt_ = t;
|
||||
}
|
||||
}
|
||||
double idleSec(double t) const { return measurable_ ? std::max(0.0, t - lastProgressAt_) : 0.0; }
|
||||
// CPU over the most recent sample interval (100 = one core), for display; < 0 unknown.
|
||||
double cpuPercent() const
|
||||
{
|
||||
if (samples_.size() < 2) return -1.0;
|
||||
const Sample& a = samples_[samples_.size() - 2];
|
||||
const Sample& b = samples_.back();
|
||||
if (a.cpu < 0 || b.cpu < 0 || b.t <= a.t) return -1.0;
|
||||
return std::max(0.0, (b.cpu - a.cpu) / (b.t - a.t) * 100.0);
|
||||
}
|
||||
|
||||
private:
|
||||
struct Sample { double t, cpu, io; };
|
||||
std::deque<Sample> samples_;
|
||||
double lastProgressAt_ = 0.0;
|
||||
bool measurable_ = false;
|
||||
};
|
||||
|
||||
enum class StopWaitDecision { Done, Wait, GiveUp, Detach, OfferForceKill, ForceKill };
|
||||
|
||||
struct StopWaitSample {
|
||||
bool processAlive = true;
|
||||
bool delivered = false;
|
||||
double elapsedSec = 0.0;
|
||||
double idleSec = 0.0;
|
||||
bool rebuildActive = false;
|
||||
StopUserChoice choice = StopUserChoice::None;
|
||||
bool killOffered = false; // the Kill offer is on screen right now
|
||||
bool shuttingDown = false; // the app began shutting down / the task was cancelled
|
||||
};
|
||||
|
||||
static inline StopWaitDecision evaluateStopWait(const StopPolicy& p, const StopWaitSample& s)
|
||||
{
|
||||
if (!s.processAlive) return StopWaitDecision::Done;
|
||||
if (p.userChoices && s.choice == StopUserChoice::Detach) return StopWaitDecision::Detach;
|
||||
if (p.userChoices && p.owned && s.choice == StopUserChoice::Kill && s.killOffered)
|
||||
return StopWaitDecision::ForceKill;
|
||||
if (p.abandonOnShutdown && s.shuttingDown) return StopWaitDecision::GiveUp;
|
||||
if (p.maxWaitSec > 0 && s.elapsedSec >= p.maxWaitSec) return StopWaitDecision::GiveUp;
|
||||
if (p.userChoices && p.owned && s.delivered && !s.rebuildActive && s.idleSec >= p.idleOfferSec)
|
||||
return StopWaitDecision::OfferForceKill;
|
||||
return StopWaitDecision::Wait;
|
||||
}
|
||||
|
||||
static constexpr int kStopTickMs = 250;
|
||||
|
||||
static inline StopOutcome executeDaemonStop(const StopPolicy& policy, StopRuntime& rt,
|
||||
LifecycleTaskContext& task)
|
||||
{
|
||||
if (!rt.processAlive()) return StopOutcome::Exited; // already gone: not a single RPC call
|
||||
|
||||
const double t0 = rt.nowSec();
|
||||
StopDeliveryPolicy delivery(policy.owned);
|
||||
StopProgressTracker progress;
|
||||
bool signalled = false;
|
||||
bool offered = false;
|
||||
auto withdrawOffer = [&] {
|
||||
if (offered) { offered = false; rt.offerForceKill(false); }
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
double now = rt.nowSec() - t0;
|
||||
if (delivery.shouldSend(now)) {
|
||||
const StopSendResult r = delivery.record(rt.trySendStop(), now);
|
||||
if (r == StopSendResult::Auth && policy.owned) {
|
||||
// Our own node refusing our credentials means they're stale (e.g. a regenerated
|
||||
// .cookie): try the next source; out of sources, the graceful signal below remains.
|
||||
if (rt.nextCredentials()) delivery.retryNow(rt.nowSec() - t0);
|
||||
else delivery.giveUp();
|
||||
}
|
||||
// POSIX: a group SIGTERM bypasses the RPC queue altogether (dragonxd's handler sets the same
|
||||
// flag as stop). Send it as soon as an attempt shows stop didn't get through.
|
||||
if (policy.gracefulSignal && !signalled && !delivery.delivered() &&
|
||||
r != StopSendResult::Queued && rt.sendGracefulSignal()) {
|
||||
signalled = true;
|
||||
delivery.markDelivered();
|
||||
}
|
||||
}
|
||||
if (!rt.processAlive()) {
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
}
|
||||
|
||||
now = rt.nowSec() - t0;
|
||||
progress.add(now, rt.cpuSeconds(), rt.ioBytes());
|
||||
StopWaitSample s;
|
||||
s.delivered = delivery.delivered();
|
||||
s.elapsedSec = now;
|
||||
s.idleSec = progress.idleSec(now);
|
||||
s.rebuildActive = rt.rebuildActive();
|
||||
s.choice = rt.userChoice();
|
||||
s.killOffered = offered;
|
||||
s.shuttingDown = task.cancelled() || task.shuttingDown();
|
||||
|
||||
switch (evaluateStopWait(policy, s)) {
|
||||
case StopWaitDecision::Done:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
case StopWaitDecision::Detach:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Detached;
|
||||
case StopWaitDecision::ForceKill:
|
||||
withdrawOffer();
|
||||
rt.forceKill();
|
||||
return StopOutcome::Killed;
|
||||
case StopWaitDecision::GiveUp:
|
||||
withdrawOffer();
|
||||
return StopOutcome::StillRunning;
|
||||
case StopWaitDecision::OfferForceKill:
|
||||
if (!offered) { offered = true; rt.offerForceKill(true); }
|
||||
break;
|
||||
case StopWaitDecision::Wait:
|
||||
withdrawOffer(); // progress resumed (or a rebuild started): the node isn't stuck after all
|
||||
break;
|
||||
}
|
||||
|
||||
StopStatus st;
|
||||
st.delivered = delivery.delivered();
|
||||
st.heard = delivery.heard();
|
||||
st.signalled = signalled;
|
||||
st.elapsedSec = now;
|
||||
st.cpuPercent = progress.cpuPercent();
|
||||
st.idleSec = s.idleSec;
|
||||
st.killOffered = offered;
|
||||
st.rebuildActive = s.rebuildActive;
|
||||
st.lastSend = delivery.last();
|
||||
st.attempts = delivery.attempts();
|
||||
rt.setStatus(st);
|
||||
task.sleepForMs(kStopTickMs);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<EmbeddedDaemon> daemon_;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user