feat(daemon): deliver stop reliably and never kill the node unasked
Closing the wallet sent "stop" at most twice, gated on a getinfo probe that needs the daemon's cs_main, then killed the node after 20 s (Windows) or 30 s (POSIX) whether or not stop had arrived. A node busy with an hour-long witness rebuild therefore never got stop, was killed mid- rebuild, and replayed everything since its last flush on every start. Delivery (src/daemon/stop_delivery.h, RPCClient::open/trySendStop): - stop goes over a dedicated client with no probe, so it never waits on rpc_'s lock or a cs_main-bound getinfo - every attempt is classified, and StopDeliveryPolicy decides the next: Accepted / 503 = delivered; Queued (reply timeout) is resent at most every 30 s; QueueFull / not listening are retried every 1.5 s; a refusal after we were heard means the node is shutting down; a 401 on our own node tries the next credential source - POSIX: a group SIGTERM goes out as soon as an attempt fails, which bypasses the HTTP queue entirely Waiting (DaemonController::executeDaemonStop, pure, virtual-clock tested): - never kills on its own. Quitting waits as long as the node needs; the shutdown screen offers "Close wallet — leave node running" after 3 s, and "Kill node (unsafe)" (confirmed) only once stop was delivered and the node showed no CPU/I-O progress for ~10 min outside a rebuild - other callers (restart, rescan, reinstall, switch, ...) wait up to 120 s, let go once the app starts quitting, and keep watching a node that is still running; wallet switch refuses to go ahead then - EmbeddedDaemon gains stop-session primitives (beginStop, pollOwnedProcess, signalGracefulStop, forceKillOwnedProcess, ownedResourceUsage over the Linux process group / Windows handle, endStop) and only one stop session runs at a time - the Force Quit button and its stall heuristic are gone; "Leave running" aborts a pending stop attempt and closes at once Verified against the fake daemon with the real GUI: closing mid-rebuild now delivers stop (or SIGTERM 2 ms after a queue-full refusal) and the node exits cleanly after its rebuild, where it was SIGKILLed before. The shutdown screen, its "Leave running" button and the idle Kill offer (after ~11 min) were checked by screenshot. New strings are English- only for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "embedded_daemon.h"
|
||||
#include "stop_delivery.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -234,6 +236,215 @@ public:
|
||||
ShutdownDecision shutdownDecision(bool keepDaemonRunning,
|
||||
bool stopExternalDaemon) const;
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Stopping the node: deliver "stop" until it lands, then wait for the process to exit. Never kill it
|
||||
// on our own: a node that is busy (a witness rebuild can hold it for an hour) or flushing would lose
|
||||
// everything since its last save and replay it on the next start. Only an explicit user "Kill",
|
||||
// offered once the node has gone idle, ends it by force; "Leave running" lets the wallet close while
|
||||
// the node finishes on its own.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
using StopUserChoice = daemon::StopUserChoice;
|
||||
using StopOutcome = daemon::StopOutcome;
|
||||
|
||||
struct StopPolicy {
|
||||
bool owned = true; // we launched it: watch the process, signal it, offer Kill
|
||||
bool gracefulSignal = false; // POSIX: a group SIGTERM is a stop channel (dragonxd treats it as stop)
|
||||
double maxWaitSec = 0.0; // stop waiting (never kill) after this long; 0 = no limit
|
||||
bool userChoices = false; // the shutdown screen offers "Leave running" / "Kill"
|
||||
bool abandonOnShutdown = false; // stop waiting once the app starts shutting down (background callers)
|
||||
double idleOfferSec = 600.0; // delivered + no CPU/I-O progress for this long: offer Kill
|
||||
};
|
||||
|
||||
// What the stop loop reports for display.
|
||||
struct StopStatus {
|
||||
bool delivered = false; // stop accepted (or the node is shutting down, or was signalled)
|
||||
bool heard = false; // the node has the request (possibly still queued)
|
||||
bool signalled = false; // a graceful signal (POSIX SIGTERM) went out
|
||||
double elapsedSec = 0.0;
|
||||
double cpuPercent = -1.0; // < 0: unknown
|
||||
double idleSec = 0.0;
|
||||
bool killOffered = false;
|
||||
bool rebuildActive = false;
|
||||
StopSendResult lastSend = StopSendResult::Other;
|
||||
int attempts = 0;
|
||||
};
|
||||
|
||||
class StopRuntime {
|
||||
public:
|
||||
virtual ~StopRuntime() = default;
|
||||
virtual double nowSec() = 0;
|
||||
virtual bool processAlive() = 0;
|
||||
virtual StopSendResult trySendStop() = 0;
|
||||
virtual bool nextCredentials() = 0; // switch to the next stop credentials; false when none left
|
||||
virtual bool sendGracefulSignal() = 0; // POSIX group SIGTERM; false where there is none (Windows)
|
||||
virtual double cpuSeconds() = 0; // cumulative CPU of the node; < 0 if unknown
|
||||
virtual double ioBytes() = 0; // cumulative I/O bytes of the node; < 0 if unknown
|
||||
virtual bool rebuildActive() = 0; // a witness rebuild is running: never offer Kill
|
||||
virtual StopUserChoice userChoice() = 0;
|
||||
virtual void offerForceKill(bool offered) = 0;
|
||||
virtual void forceKill() = 0;
|
||||
virtual void setStatus(const StopStatus& status) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* "Making progress" = >= 0.05 cores of CPU or >= 1 MB of I/O over the last >= 60s. A flush that
|
||||
* fsyncs a big wallet is mostly I/O at low CPU; a rebuild is mostly CPU. Idle time counts from the
|
||||
* first sample. With no readings at all (unknown), it never reports idle, so Kill is never offered.
|
||||
*/
|
||||
class StopProgressTracker {
|
||||
public:
|
||||
static constexpr double kWindowSec = 60.0;
|
||||
static constexpr double kMinCores = 0.05;
|
||||
static constexpr double kMinIoBytes = 1024.0 * 1024.0;
|
||||
|
||||
void add(double t, double cpuSec, double ioBytes)
|
||||
{
|
||||
if (cpuSec >= 0 || ioBytes >= 0) measurable_ = true;
|
||||
if (samples_.empty()) lastProgressAt_ = t;
|
||||
samples_.push_back({t, cpuSec, ioBytes});
|
||||
// Keep the newest sample that is at least a full window old as the comparison base.
|
||||
while (samples_.size() > 2 && samples_[1].t <= t - kWindowSec) samples_.pop_front();
|
||||
const Sample& base = samples_.front();
|
||||
const double span = t - base.t;
|
||||
if (span >= kWindowSec) {
|
||||
const bool cpuMoved = cpuSec >= 0 && base.cpu >= 0 && (cpuSec - base.cpu) / span >= kMinCores;
|
||||
const bool ioMoved = ioBytes >= 0 && base.io >= 0 && (ioBytes - base.io) >= kMinIoBytes;
|
||||
if (cpuMoved || ioMoved) lastProgressAt_ = t;
|
||||
}
|
||||
}
|
||||
double idleSec(double t) const { return measurable_ ? std::max(0.0, t - lastProgressAt_) : 0.0; }
|
||||
// CPU over the most recent sample interval (100 = one core), for display; < 0 unknown.
|
||||
double cpuPercent() const
|
||||
{
|
||||
if (samples_.size() < 2) return -1.0;
|
||||
const Sample& a = samples_[samples_.size() - 2];
|
||||
const Sample& b = samples_.back();
|
||||
if (a.cpu < 0 || b.cpu < 0 || b.t <= a.t) return -1.0;
|
||||
return std::max(0.0, (b.cpu - a.cpu) / (b.t - a.t) * 100.0);
|
||||
}
|
||||
|
||||
private:
|
||||
struct Sample { double t, cpu, io; };
|
||||
std::deque<Sample> samples_;
|
||||
double lastProgressAt_ = 0.0;
|
||||
bool measurable_ = false;
|
||||
};
|
||||
|
||||
enum class StopWaitDecision { Done, Wait, GiveUp, Detach, OfferForceKill, ForceKill };
|
||||
|
||||
struct StopWaitSample {
|
||||
bool processAlive = true;
|
||||
bool delivered = false;
|
||||
double elapsedSec = 0.0;
|
||||
double idleSec = 0.0;
|
||||
bool rebuildActive = false;
|
||||
StopUserChoice choice = StopUserChoice::None;
|
||||
bool killOffered = false; // the Kill offer is on screen right now
|
||||
bool shuttingDown = false; // the app began shutting down / the task was cancelled
|
||||
};
|
||||
|
||||
static inline StopWaitDecision evaluateStopWait(const StopPolicy& p, const StopWaitSample& s)
|
||||
{
|
||||
if (!s.processAlive) return StopWaitDecision::Done;
|
||||
if (p.userChoices && s.choice == StopUserChoice::Detach) return StopWaitDecision::Detach;
|
||||
if (p.userChoices && p.owned && s.choice == StopUserChoice::Kill && s.killOffered)
|
||||
return StopWaitDecision::ForceKill;
|
||||
if (p.abandonOnShutdown && s.shuttingDown) return StopWaitDecision::GiveUp;
|
||||
if (p.maxWaitSec > 0 && s.elapsedSec >= p.maxWaitSec) return StopWaitDecision::GiveUp;
|
||||
if (p.userChoices && p.owned && s.delivered && !s.rebuildActive && s.idleSec >= p.idleOfferSec)
|
||||
return StopWaitDecision::OfferForceKill;
|
||||
return StopWaitDecision::Wait;
|
||||
}
|
||||
|
||||
static constexpr int kStopTickMs = 250;
|
||||
|
||||
static inline StopOutcome executeDaemonStop(const StopPolicy& policy, StopRuntime& rt,
|
||||
LifecycleTaskContext& task)
|
||||
{
|
||||
if (!rt.processAlive()) return StopOutcome::Exited; // already gone: not a single RPC call
|
||||
|
||||
const double t0 = rt.nowSec();
|
||||
StopDeliveryPolicy delivery(policy.owned);
|
||||
StopProgressTracker progress;
|
||||
bool signalled = false;
|
||||
bool offered = false;
|
||||
auto withdrawOffer = [&] {
|
||||
if (offered) { offered = false; rt.offerForceKill(false); }
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
double now = rt.nowSec() - t0;
|
||||
if (delivery.shouldSend(now)) {
|
||||
const StopSendResult r = delivery.record(rt.trySendStop(), now);
|
||||
if (r == StopSendResult::Auth && policy.owned) {
|
||||
// Our own node refusing our credentials means they're stale (e.g. a regenerated
|
||||
// .cookie): try the next source; out of sources, the graceful signal below remains.
|
||||
if (rt.nextCredentials()) delivery.retryNow(rt.nowSec() - t0);
|
||||
else delivery.giveUp();
|
||||
}
|
||||
// POSIX: a group SIGTERM bypasses the RPC queue altogether (dragonxd's handler sets the same
|
||||
// flag as stop). Send it as soon as an attempt shows stop didn't get through.
|
||||
if (policy.gracefulSignal && !signalled && !delivery.delivered() &&
|
||||
r != StopSendResult::Queued && rt.sendGracefulSignal()) {
|
||||
signalled = true;
|
||||
delivery.markDelivered();
|
||||
}
|
||||
}
|
||||
if (!rt.processAlive()) {
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
}
|
||||
|
||||
now = rt.nowSec() - t0;
|
||||
progress.add(now, rt.cpuSeconds(), rt.ioBytes());
|
||||
StopWaitSample s;
|
||||
s.delivered = delivery.delivered();
|
||||
s.elapsedSec = now;
|
||||
s.idleSec = progress.idleSec(now);
|
||||
s.rebuildActive = rt.rebuildActive();
|
||||
s.choice = rt.userChoice();
|
||||
s.killOffered = offered;
|
||||
s.shuttingDown = task.cancelled() || task.shuttingDown();
|
||||
|
||||
switch (evaluateStopWait(policy, s)) {
|
||||
case StopWaitDecision::Done:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
case StopWaitDecision::Detach:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Detached;
|
||||
case StopWaitDecision::ForceKill:
|
||||
withdrawOffer();
|
||||
rt.forceKill();
|
||||
return StopOutcome::Killed;
|
||||
case StopWaitDecision::GiveUp:
|
||||
withdrawOffer();
|
||||
return StopOutcome::StillRunning;
|
||||
case StopWaitDecision::OfferForceKill:
|
||||
if (!offered) { offered = true; rt.offerForceKill(true); }
|
||||
break;
|
||||
case StopWaitDecision::Wait:
|
||||
withdrawOffer(); // progress resumed (or a rebuild started): the node isn't stuck after all
|
||||
break;
|
||||
}
|
||||
|
||||
StopStatus st;
|
||||
st.delivered = delivery.delivered();
|
||||
st.heard = delivery.heard();
|
||||
st.signalled = signalled;
|
||||
st.elapsedSec = now;
|
||||
st.cpuPercent = progress.cpuPercent();
|
||||
st.idleSec = s.idleSec;
|
||||
st.killOffered = offered;
|
||||
st.rebuildActive = s.rebuildActive;
|
||||
st.lastSend = delivery.last();
|
||||
st.attempts = delivery.attempts();
|
||||
rt.setStatus(st);
|
||||
task.sleepForMs(kStopTickMs);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<EmbeddedDaemon> daemon_;
|
||||
};
|
||||
|
||||
@@ -948,6 +948,78 @@ bool EmbeddedDaemon::stop(int wait_ms)
|
||||
return exitedOnItsOwn;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::beginStop()
|
||||
{
|
||||
stop_requested_ = true;
|
||||
should_stop_ = true;
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
if (!isRunning()) return false; // handle-based: our own process only
|
||||
setState(State::Stopping, "Stopping dragonxd...");
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::pollOwnedProcess()
|
||||
{
|
||||
drainOutput();
|
||||
return isRunning();
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::signalGracefulStop()
|
||||
{
|
||||
return false; // no graceful channel on Windows: the GUI launches dragonxd without a console
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::forceKillOwnedProcess()
|
||||
{
|
||||
if (process_handle_ == nullptr || !isRunning()) return;
|
||||
DEBUG_LOGF("Killing dragonxd at the user's request (TerminateProcess on our own handle)\n");
|
||||
TerminateProcess(process_handle_, 1);
|
||||
WaitForSingleObject(process_handle_, 5000);
|
||||
drainOutput();
|
||||
}
|
||||
|
||||
EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const
|
||||
{
|
||||
ResourceUsage usage;
|
||||
if (process_handle_ == nullptr) return usage;
|
||||
FILETIME created, exited, kernel, user;
|
||||
if (GetProcessTimes(process_handle_, &created, &exited, &kernel, &user)) {
|
||||
auto ticks = [](const FILETIME& ft) {
|
||||
return (static_cast<unsigned long long>(ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
|
||||
};
|
||||
usage.cpuSeconds = static_cast<double>(ticks(kernel) + ticks(user)) / 1e7; // 100ns units
|
||||
}
|
||||
IO_COUNTERS io;
|
||||
if (GetProcessIoCounters(process_handle_, &io))
|
||||
usage.ioBytes = static_cast<double>(io.ReadTransferCount + io.WriteTransferCount);
|
||||
return usage;
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::endStop(bool keepWatching)
|
||||
{
|
||||
if (keepWatching && isRunning()) {
|
||||
// Still running and still ours: watch it again. Its exit is booked as a clean stop.
|
||||
should_stop_ = false;
|
||||
monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
|
||||
return;
|
||||
}
|
||||
const bool leftRunning = isRunning();
|
||||
if (leftRunning)
|
||||
DEBUG_LOGF("Leaving dragonxd running; it will finish shutting down on its own\n");
|
||||
if (process_handle_ != nullptr) {
|
||||
CloseHandle(process_handle_); // only our handle: the process carries on
|
||||
process_handle_ = nullptr;
|
||||
}
|
||||
if (stdout_read_ != nullptr) {
|
||||
CloseHandle(stdout_read_);
|
||||
stdout_read_ = nullptr;
|
||||
}
|
||||
drainOutput();
|
||||
setState(State::Stopped, leftRunning ? "dragonxd left running" : "dragonxd stopped");
|
||||
}
|
||||
|
||||
// Translate Windows NTSTATUS / exit codes to human-readable descriptions
|
||||
static std::string translateWindowsExitCode(DWORD code) {
|
||||
switch (code) {
|
||||
@@ -1396,6 +1468,125 @@ bool EmbeddedDaemon::stop(int wait_ms)
|
||||
return exitedOnItsOwn;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::beginStop()
|
||||
{
|
||||
stop_requested_ = true;
|
||||
should_stop_ = true;
|
||||
// The monitor sees should_stop_ within one 100ms tick. If the child exited in that window the
|
||||
// monitor reaped it (as a clean stop: stop_requested_ is set) and the pid is now 0.
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
if (process_pid_ <= 0) return false;
|
||||
setState(State::Stopping, "Stopping dragonxd...");
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::pollOwnedProcess()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return false;
|
||||
drainOutput();
|
||||
int status = 0;
|
||||
const pid_t r = waitpid(pid, &status, WNOHANG);
|
||||
if (r == 0 || (r < 0 && errno == EINTR)) return true;
|
||||
if (r == pid) DEBUG_LOGF("dragonxd exited (%s)\n", describeExitStatus(status).c_str());
|
||||
drainOutput();
|
||||
process_pid_ = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::signalGracefulStop()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return false;
|
||||
// The whole group: with a bash wrapper (embedded_daemon.cpp startProcess) a SIGTERM to the wrapper
|
||||
// alone would orphan dragonxd.
|
||||
DEBUG_LOGF("Sending SIGTERM to dragonxd's process group -%d (graceful stop)\n", pid);
|
||||
return kill(-pid, SIGTERM) == 0;
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::forceKillOwnedProcess()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return;
|
||||
DEBUG_LOGF("Killing dragonxd (group -%d) at the user's request\n", pid);
|
||||
kill(-pid, SIGKILL);
|
||||
int status = 0;
|
||||
while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {}
|
||||
drainOutput();
|
||||
process_pid_ = 0;
|
||||
}
|
||||
|
||||
EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const
|
||||
{
|
||||
ResourceUsage usage;
|
||||
#ifdef __APPLE__
|
||||
return usage; // no sampler yet (would need proc_pid_rusage): never "idle", so Kill is never offered
|
||||
#else
|
||||
const pid_t pgid = process_pid_;
|
||||
if (pgid <= 0) return usage;
|
||||
// The tracked pid may be a bash wrapper that itself uses ~0 CPU: sum the whole process group.
|
||||
unsigned long long ticks = 0;
|
||||
double io = 0.0;
|
||||
bool anyCpu = false, anyIo = false;
|
||||
std::error_code ec;
|
||||
fs::directory_iterator it("/proc", ec), end;
|
||||
for (; !ec && it != end; it.increment(ec)) {
|
||||
const std::string name = it->path().filename().string();
|
||||
if (name.empty() || name[0] < '0' || name[0] > '9') continue;
|
||||
std::ifstream statFile((it->path() / "stat").string());
|
||||
std::string line;
|
||||
long pgrp = 0;
|
||||
unsigned long long t = 0;
|
||||
if (!statFile || !std::getline(statFile, line) || !parseProcStatLine(line, pgrp, t) || pgrp != pgid)
|
||||
continue;
|
||||
ticks += t;
|
||||
anyCpu = true;
|
||||
std::ifstream ioFile((it->path() / "io").string());
|
||||
std::string key;
|
||||
unsigned long long value = 0;
|
||||
while (ioFile >> key >> value) {
|
||||
if (key == "read_bytes:" || key == "write_bytes:") {
|
||||
io += static_cast<double>(value);
|
||||
anyIo = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
const long hz = sysconf(_SC_CLK_TCK);
|
||||
if (anyCpu && hz > 0) usage.cpuSeconds = static_cast<double>(ticks) / static_cast<double>(hz);
|
||||
if (anyIo) usage.ioBytes = io;
|
||||
return usage;
|
||||
#endif
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::endStop(bool keepWatching)
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid > 0 && keepWatching) {
|
||||
// Still running and still ours: watch it again. Its exit is booked as a clean stop.
|
||||
should_stop_ = false;
|
||||
monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
|
||||
return;
|
||||
}
|
||||
if (pid > 0) {
|
||||
// Leave it running. Close our end of its output pipe: its further writes fail with EPIPE, which
|
||||
// dragonxd ignores (it ignores SIGPIPE), instead of blocking once a pipe nobody drains is full.
|
||||
// A detached waiter reaps it when it exits, if this process is still around then.
|
||||
DEBUG_LOGF("Leaving dragonxd (pid %d) running; it will finish shutting down on its own\n", pid);
|
||||
std::thread([pid] {
|
||||
int st = 0;
|
||||
while (waitpid(pid, &st, 0) < 0 && errno == EINTR) {}
|
||||
}).detach();
|
||||
process_pid_ = 0;
|
||||
}
|
||||
if (stdout_fd_ >= 0) {
|
||||
close(stdout_fd_);
|
||||
stdout_fd_ = -1;
|
||||
}
|
||||
setState(State::Stopped, pid > 0 ? "dragonxd left running" : "dragonxd stopped");
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::monitorProcess()
|
||||
{
|
||||
char buffer[4096];
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -71,6 +72,47 @@ public:
|
||||
*/
|
||||
void markStopRequested() { stop_requested_ = true; }
|
||||
|
||||
// --- A stop session, driven by DaemonController::executeDaemonStop through the app's StopRuntime ---
|
||||
|
||||
/** Marks the stop as requested and stops the monitor, so the session is the only reaper.
|
||||
* @return true if a process this object launched is still alive; then finish with endStop(). */
|
||||
bool beginStop();
|
||||
/** Polls our own process (reaping it on POSIX) and drains its output; false once it has exited. */
|
||||
bool pollOwnedProcess();
|
||||
/** POSIX: SIGTERM our process group (dragonxd treats it like RPC stop). Windows has no such channel. */
|
||||
bool signalGracefulStop();
|
||||
/** Ends our own process by force (SIGKILL / TerminateProcess) and reaps it. Explicit user choice only. */
|
||||
void forceKillOwnedProcess();
|
||||
struct ResourceUsage {
|
||||
double cpuSeconds = -1.0; // cumulative; < 0 if unknown
|
||||
double ioBytes = -1.0; // cumulative storage I/O; < 0 if unknown
|
||||
};
|
||||
/** CPU and I/O used so far by our process (its whole process group on Linux). */
|
||||
ResourceUsage ownedResourceUsage() const;
|
||||
/** Finishes a stop session. The process is gone: release everything. Still running: keepWatching
|
||||
* resumes the monitor (its exit is still seen, and booked as a clean stop); otherwise let it go —
|
||||
* the user chose to leave it running. */
|
||||
void endStop(bool keepWatching);
|
||||
|
||||
/**
|
||||
* @brief Parse a /proc/<pid>/stat line: process group (field 5) and utime+stime ticks (fields 14+15)
|
||||
*
|
||||
* comm (field 2) is parenthesised and may itself contain spaces and ')', so fields are read from
|
||||
* after the LAST ')'.
|
||||
*/
|
||||
static bool parseProcStatLine(const std::string& line, long& pgrp, unsigned long long& cpuTicks)
|
||||
{
|
||||
const size_t close = line.rfind(')');
|
||||
if (close == std::string::npos) return false;
|
||||
unsigned long long utime = 0, stime = 0;
|
||||
// state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt cmajflt utime stime
|
||||
if (std::sscanf(line.c_str() + close + 1, " %*c %*d %ld %*d %*d %*d %*u %*u %*u %*u %*u %llu %llu",
|
||||
&pgrp, &utime, &stime) != 3)
|
||||
return false;
|
||||
cpuTicks = utime + stime;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Check if daemon process is running
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
#pragma once
|
||||
|
||||
// Delivering "stop" to dragonxd: what one attempt achieved, and when to send another. Pure and
|
||||
// header-only so the test target can drive it without the RPC client.
|
||||
//
|
||||
// Why retrying matters: dragonxd's HTTP server has 8 workers and a 16-deep queue. While a long
|
||||
// operation holds cs_main, the GUI's own abandoned polls can fill every slot, and each new request is
|
||||
// refused at once with "Work queue depth exceeded" until a slot frees. The old code made one or two
|
||||
// attempts, gated on a getinfo probe that needs cs_main, so stop was never delivered and the node was
|
||||
// then killed.
|
||||
|
||||
#include "rpc/rpc_failure.h"
|
||||
|
||||
namespace dragonx {
|
||||
namespace daemon {
|
||||
|
||||
// What the user picked on the shutdown screen while the node was stopping.
|
||||
enum class StopUserChoice { None, Detach, Kill };
|
||||
|
||||
// How a stop session ended. The wallet never kills the node unless the user chose Kill.
|
||||
enum class StopOutcome {
|
||||
Exited, // the process is gone
|
||||
StillRunning, // we stopped waiting (time budget, app shutdown, cancellation); never killed
|
||||
Detached, // the user chose to leave it running
|
||||
Killed, // the user chose Kill
|
||||
};
|
||||
|
||||
inline const char* stopOutcomeName(StopOutcome o)
|
||||
{
|
||||
switch (o) {
|
||||
case StopOutcome::Exited: return "Exited";
|
||||
case StopOutcome::StillRunning: return "StillRunning";
|
||||
case StopOutcome::Detached: return "Detached";
|
||||
case StopOutcome::Killed: return "Killed";
|
||||
}
|
||||
return "Exited";
|
||||
}
|
||||
|
||||
enum class StopSendResult {
|
||||
Accepted, // HTTP 200: the daemon ran stop
|
||||
Queued, // reply timeout: it reached the daemon and waits for a worker; it will run
|
||||
QueueFull, // "Work queue depth exceeded": refused before queueing (holds no slot)
|
||||
NotListening, // refused / connect timeout: RPC isn't bound (not up yet, or already unbound)
|
||||
ShuttingDown, // 503 / "Unhandled request": the daemon is already shutting down
|
||||
Auth, // 401: our credentials are wrong or stale
|
||||
Other,
|
||||
};
|
||||
|
||||
inline const char* stopSendResultName(StopSendResult r)
|
||||
{
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted: return "Accepted";
|
||||
case StopSendResult::Queued: return "Queued";
|
||||
case StopSendResult::QueueFull: return "QueueFull";
|
||||
case StopSendResult::NotListening: return "NotListening";
|
||||
case StopSendResult::ShuttingDown: return "ShuttingDown";
|
||||
case StopSendResult::Auth: return "Auth";
|
||||
case StopSendResult::Other: return "Other";
|
||||
}
|
||||
return "Other";
|
||||
}
|
||||
|
||||
// What a stop call's failure kind (RpcFailureKind::None = HTTP 200) means for delivery.
|
||||
inline StopSendResult stopSendResultFromKind(rpc::RpcFailureKind kind)
|
||||
{
|
||||
using K = rpc::RpcFailureKind;
|
||||
switch (kind) {
|
||||
case K::None: return StopSendResult::Accepted;
|
||||
case K::ReplyTimeout: return StopSendResult::Queued;
|
||||
case K::QueueFull: return StopSendResult::QueueFull;
|
||||
case K::NotListening:
|
||||
case K::ConnectTimeout: return StopSendResult::NotListening;
|
||||
case K::ShuttingDown:
|
||||
case K::Unhandled: return StopSendResult::ShuttingDown;
|
||||
case K::Auth: return StopSendResult::Auth;
|
||||
default: return StopSendResult::Other;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When to (re)send stop. Times are seconds on the caller's clock.
|
||||
*
|
||||
* - Accepted / ShuttingDown: delivered, never resend.
|
||||
* - Queued: the daemon has it. Resend at most every 30s: each resend takes another queue slot, and
|
||||
* concurrent stops race on the miner threads (stop's GenerateBitcoins(false)).
|
||||
* - QueueFull / NotListening / Other: not delivered, cheap to retry: again after 1.5s.
|
||||
* - NotListening AFTER the daemon had heard us means it unbound its listener to shut down: delivered.
|
||||
* - Auth: give up for a node we don't own. For our own node the caller may switch credentials and
|
||||
* retry at once (retryNow), or give up (giveUp).
|
||||
*/
|
||||
class StopDeliveryPolicy {
|
||||
public:
|
||||
static constexpr double kRetrySec = 1.5;
|
||||
static constexpr double kQueuedResendSec = 30.0;
|
||||
|
||||
explicit StopDeliveryPolicy(bool owned) : owned_(owned) {}
|
||||
|
||||
bool shouldSend(double now) const { return !finished_ && now >= nextSendAt_; }
|
||||
|
||||
// Records one attempt; returns it as interpreted (a refusal after we were heard reads as ShuttingDown).
|
||||
StopSendResult record(StopSendResult r, double now)
|
||||
{
|
||||
++attempts_;
|
||||
if (r == StopSendResult::NotListening && heard_) r = StopSendResult::ShuttingDown;
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted:
|
||||
heard_ = true;
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::ShuttingDown:
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::Queued:
|
||||
heard_ = true;
|
||||
nextSendAt_ = now + kQueuedResendSec;
|
||||
break;
|
||||
case StopSendResult::Auth:
|
||||
authRejected_ = true;
|
||||
if (!owned_) finished_ = true;
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
case StopSendResult::QueueFull:
|
||||
case StopSendResult::NotListening:
|
||||
case StopSendResult::Other:
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
}
|
||||
last_ = r;
|
||||
return r;
|
||||
}
|
||||
|
||||
void markDelivered() { delivered_ = finished_ = true; } // also for a graceful signal (POSIX SIGTERM)
|
||||
void retryNow(double now) { nextSendAt_ = now; }
|
||||
void giveUp() { finished_ = true; }
|
||||
|
||||
bool delivered() const { return delivered_; } // the daemon is stopping (or was signalled)
|
||||
bool heard() const { return heard_; } // it has (or ran) our request
|
||||
bool finished() const { return finished_; } // no more attempts
|
||||
bool authRejected() const { return authRejected_; }
|
||||
int attempts() const { return attempts_; }
|
||||
StopSendResult last() const { return last_; }
|
||||
|
||||
private:
|
||||
bool owned_ = true;
|
||||
bool delivered_ = false;
|
||||
bool heard_ = false;
|
||||
bool finished_ = false;
|
||||
bool authRejected_ = false;
|
||||
int attempts_ = 0;
|
||||
double nextSendAt_ = 0.0;
|
||||
StopSendResult last_ = StopSendResult::Other;
|
||||
};
|
||||
|
||||
} // namespace daemon
|
||||
} // namespace dragonx
|
||||
Reference in New Issue
Block a user