feat(daemon): deliver stop reliably and never kill the node unasked
Closing the wallet sent "stop" at most twice, gated on a getinfo probe that needs the daemon's cs_main, then killed the node after 20 s (Windows) or 30 s (POSIX) whether or not stop had arrived. A node busy with an hour-long witness rebuild therefore never got stop, was killed mid- rebuild, and replayed everything since its last flush on every start. Delivery (src/daemon/stop_delivery.h, RPCClient::open/trySendStop): - stop goes over a dedicated client with no probe, so it never waits on rpc_'s lock or a cs_main-bound getinfo - every attempt is classified, and StopDeliveryPolicy decides the next: Accepted / 503 = delivered; Queued (reply timeout) is resent at most every 30 s; QueueFull / not listening are retried every 1.5 s; a refusal after we were heard means the node is shutting down; a 401 on our own node tries the next credential source - POSIX: a group SIGTERM goes out as soon as an attempt fails, which bypasses the HTTP queue entirely Waiting (DaemonController::executeDaemonStop, pure, virtual-clock tested): - never kills on its own. Quitting waits as long as the node needs; the shutdown screen offers "Close wallet — leave node running" after 3 s, and "Kill node (unsafe)" (confirmed) only once stop was delivered and the node showed no CPU/I-O progress for ~10 min outside a rebuild - other callers (restart, rescan, reinstall, switch, ...) wait up to 120 s, let go once the app starts quitting, and keep watching a node that is still running; wallet switch refuses to go ahead then - EmbeddedDaemon gains stop-session primitives (beginStop, pollOwnedProcess, signalGracefulStop, forceKillOwnedProcess, ownedResourceUsage over the Linux process group / Windows handle, endStop) and only one stop session runs at a time - the Force Quit button and its stall heuristic are gone; "Leave running" aborts a pending stop attempt and closes at once Verified against the fake daemon with the real GUI: closing mid-rebuild now delivers stop (or SIGTERM 2 ms after a queue-full refusal) and the node exits cleanly after its rebuild, where it was SIGKILLed before. The shutdown screen, its "Leave running" button and the idle Kill offer (after ~11 min) were checked by screenshot. New strings are English- only for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+274
-140
@@ -5378,105 +5378,231 @@ bool App::startEmbeddedDaemon()
|
||||
return daemon_controller_->start(settings_.get());
|
||||
}
|
||||
|
||||
void App::stopEmbeddedDaemon()
|
||||
// Drives one stop session for DaemonController::executeDaemonStop: a dedicated RPC client for "stop" (not
|
||||
// rpc_, which another thread may hold, and whose getinfo probe needs the daemon's cs_main), the owned
|
||||
// process's exit / signal / CPU, and the shutdown screen's choices.
|
||||
class AppStopRuntime final : public daemon::DaemonController::StopRuntime {
|
||||
public:
|
||||
using DC = daemon::DaemonController;
|
||||
|
||||
AppStopRuntime(App& app, daemon::EmbeddedDaemon* owned, std::vector<rpc::ConnectionConfig> credentials)
|
||||
: app_(app), owned_(owned), creds_(std::move(credentials)), t0_(std::chrono::steady_clock::now())
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(app_.stop_client_mutex_);
|
||||
app_.active_stop_client_ = &client_;
|
||||
}
|
||||
~AppStopRuntime() override
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(app_.stop_client_mutex_);
|
||||
app_.active_stop_client_ = nullptr;
|
||||
}
|
||||
|
||||
double nowSec() override
|
||||
{
|
||||
return std::chrono::duration<double>(std::chrono::steady_clock::now() - t0_).count();
|
||||
}
|
||||
bool processAlive() override
|
||||
{
|
||||
if (owned_) return owned_->pollOwnedProcess();
|
||||
// Not ours: we can only watch for it to go (port and process), never wait on or signal it.
|
||||
return daemon::EmbeddedDaemon::isRpcPortInUse() || daemon::EmbeddedDaemon::isDaemonProcessRunning();
|
||||
}
|
||||
daemon::StopSendResult trySendStop() override
|
||||
{
|
||||
if (creds_.empty()) return daemon::StopSendResult::Other;
|
||||
if (!clientOpen_) {
|
||||
const auto& c = creds_[credIndex_];
|
||||
clientOpen_ = client_.open(c.host, c.port, c.rpcuser, c.rpcpassword, c.use_tls);
|
||||
if (!clientOpen_) return daemon::StopSendResult::Other;
|
||||
}
|
||||
const rpc::RpcFailureKind kind = client_.trySendStop(2);
|
||||
const daemon::StopSendResult r = daemon::stopSendResultFromKind(kind);
|
||||
++attempts_;
|
||||
if (attempts_ == 1 || r != lastLogged_) { // log changes, not every retry
|
||||
DEBUG_LOGF("[stop] attempt %d at %.1fs: %s (%s)\n", attempts_, nowSec(),
|
||||
daemon::stopSendResultName(r), rpc::rpcFailureKindName(kind));
|
||||
lastLogged_ = r;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
bool nextCredentials() override
|
||||
{
|
||||
if (credIndex_ + 1 >= creds_.size()) return false;
|
||||
++credIndex_;
|
||||
clientOpen_ = false;
|
||||
DEBUG_LOGF("[stop] HTTP 401 — retrying with the next credential source (%zu of %zu)\n",
|
||||
credIndex_ + 1, creds_.size());
|
||||
return true;
|
||||
}
|
||||
bool sendGracefulSignal() override { return owned_ && owned_->signalGracefulStop(); }
|
||||
double cpuSeconds() override
|
||||
{
|
||||
usage_ = owned_ ? owned_->ownedResourceUsage() : daemon::EmbeddedDaemon::ResourceUsage{};
|
||||
return usage_.cpuSeconds;
|
||||
}
|
||||
double ioBytes() override { return usage_.ioBytes; } // sampled with cpuSeconds()
|
||||
bool rebuildActive() override { return false; } // the witness-rebuild tracker is not wired yet
|
||||
daemon::StopUserChoice userChoice() override
|
||||
{
|
||||
return static_cast<daemon::StopUserChoice>(app_.stop_user_choice_.load());
|
||||
}
|
||||
void offerForceKill(bool offered) override
|
||||
{
|
||||
DEBUG_LOGF("[stop] %s\n", offered ? "no CPU/I-O progress for a long time: offering Kill"
|
||||
: "progress resumed: Kill offer withdrawn");
|
||||
app_.stop_kill_offered_ = offered;
|
||||
}
|
||||
void forceKill() override
|
||||
{
|
||||
DEBUG_LOGF("[stop] user chose Kill after %.0fs\n", nowSec());
|
||||
if (owned_) owned_->forceKillOwnedProcess();
|
||||
}
|
||||
void setStatus(const DC::StopStatus& s) override
|
||||
{
|
||||
if (s.delivered && !loggedDelivered_) {
|
||||
DEBUG_LOGF("[stop] stop delivered at %.1fs (%s)\n", s.elapsedSec,
|
||||
s.signalled ? "SIGTERM to the process group" : daemon::stopSendResultName(s.lastSend));
|
||||
loggedDelivered_ = true;
|
||||
}
|
||||
std::string text = TR(s.delivered ? "shutdown_stop_delivered"
|
||||
: s.heard ? "shutdown_stop_queued"
|
||||
: "shutdown_stop_asking");
|
||||
if (s.cpuPercent >= 0) {
|
||||
char buf[64];
|
||||
snprintf(buf, sizeof(buf), TR("shutdown_stop_cpu_suffix"), static_cast<int>(s.cpuPercent + 0.5));
|
||||
text += buf;
|
||||
}
|
||||
app_.shutdown_status_ = text;
|
||||
app_.stop_idle_minutes_ = static_cast<int>(s.idleSec / 60.0);
|
||||
}
|
||||
|
||||
private:
|
||||
App& app_;
|
||||
daemon::EmbeddedDaemon* owned_ = nullptr;
|
||||
std::vector<rpc::ConnectionConfig> creds_;
|
||||
size_t credIndex_ = 0;
|
||||
rpc::RPCClient client_;
|
||||
bool clientOpen_ = false;
|
||||
int attempts_ = 0;
|
||||
daemon::StopSendResult lastLogged_ = daemon::StopSendResult::Other;
|
||||
bool loggedDelivered_ = false;
|
||||
daemon::EmbeddedDaemon::ResourceUsage usage_;
|
||||
std::chrono::steady_clock::time_point t0_;
|
||||
};
|
||||
|
||||
class AppStopTaskContext final : public daemon::DaemonController::LifecycleTaskContext {
|
||||
public:
|
||||
AppStopTaskContext(App& app, bool watchShutdown) : app_(app), watchShutdown_(watchShutdown) {}
|
||||
bool cancelled() const override { return false; }
|
||||
bool shuttingDown() const override { return watchShutdown_ && app_.shutting_down_.load(); }
|
||||
void sleepForMs(int ms) override { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); }
|
||||
|
||||
private:
|
||||
App& app_;
|
||||
bool watchShutdown_ = false;
|
||||
};
|
||||
|
||||
void App::chooseStopAction(daemon::StopUserChoice choice)
|
||||
{
|
||||
if (!daemon_controller_) return;
|
||||
stop_user_choice_ = static_cast<int>(choice);
|
||||
if (choice == daemon::StopUserChoice::Detach) {
|
||||
// End a pending stop attempt now rather than after its timeout.
|
||||
std::lock_guard<std::mutex> lk(stop_client_mutex_);
|
||||
if (active_stop_client_) active_stop_client_->requestAbort();
|
||||
}
|
||||
}
|
||||
|
||||
daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode)
|
||||
{
|
||||
if (!daemon_controller_) return daemon::StopOutcome::Exited;
|
||||
|
||||
auto shutdownDecision = daemon_controller_->shutdownDecision(
|
||||
false, settings_ && settings_->getStopExternalDaemon());
|
||||
if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) {
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s — skipping\n", shutdownDecision.logReason);
|
||||
return;
|
||||
return daemon::StopOutcome::StillRunning;
|
||||
}
|
||||
|
||||
// Do we hold a live process handle for this node? Capture it BEFORE stop() reaps the pid. If owned,
|
||||
// daemon_controller_->stop() below BLOCKS until the process actually exits. If NOT owned (external /
|
||||
// adopted / direct-connected / a lock-gated start that never spawned), stop() returns at once on every
|
||||
// platform — it only ever waits on (or kills) a process this app launched, never one found by name —
|
||||
// so we can only ask it over RPC and then watch for it to disappear (handled after the stop() call).
|
||||
const bool owned = daemon_controller_->isRunning();
|
||||
// One stop at a time: a second caller (say the shutdown thread while a restart task is stopping the
|
||||
// node) waits for the first. Background stops let go as soon as the app starts quitting, so the
|
||||
// shutdown never waits long behind one.
|
||||
std::lock_guard<std::mutex> single(stop_mutex_);
|
||||
daemon::EmbeddedDaemon* d = daemon_controller_->daemon();
|
||||
// beginStop() also marks the stop as requested (an early exit is not a crash) and makes this session
|
||||
// the only reaper. Only a process we launched and still hold is "owned": a lock-gated start, an
|
||||
// external node and a node we already let go of are not, and are never waited on, signalled or killed.
|
||||
const bool owned = d && d->beginStop();
|
||||
if (d && !owned) d->endStop(/*keepWatching=*/false);
|
||||
|
||||
// Any exit from here on is the stop we are asking for, not a crash — even one that lands before
|
||||
// daemon_controller_->stop() below gets to reap it (crash_count_ drives auto-restart / give-up).
|
||||
daemon_controller_->markStopRequested();
|
||||
if (!owned && !daemon::EmbeddedDaemon::isRpcPortInUse()) {
|
||||
// Nothing listening for a stop: either it is gone, or it is another dragonxd we can't talk to (one
|
||||
// still finishing its own shutdown trips the start() lock gate this way). Leave it be.
|
||||
const bool running = daemon::EmbeddedDaemon::isDaemonProcessRunning();
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: no node of ours and no RPC listener%s\n",
|
||||
running ? " — another dragonxd is running; leaving it alone" : "");
|
||||
return running ? daemon::StopOutcome::StillRunning : daemon::StopOutcome::Exited;
|
||||
}
|
||||
|
||||
// Send RPC "stop" command — this is the graceful path that lets the
|
||||
// daemon flush state, save block indexes, close sockets, etc.
|
||||
bool stop_sent = false;
|
||||
|
||||
// Try the existing RPC connection first. beginShutdown() called rpc_->requestAbort() to unblock any
|
||||
// in-flight request — but that sticky abort (cleared only by connect()) would make THIS "stop" self-
|
||||
// abort mid-flight (CURLE_ABORTED_BY_CALLBACK). The old async rpc_->stop() swallowed that error yet
|
||||
// set stop_sent=true, so the daemon never heard "stop" and the working temp-connection fallback was
|
||||
// skipped — which is why "Stop external daemon" left an external dragonxd running (it only died via
|
||||
// the 20s force-kill). Clear the abort and send SYNCHRONOUSLY so real success is surfaced.
|
||||
if (rpc_ && rpc_->isConnected()) {
|
||||
DEBUG_LOGF("Sending stop command via existing RPC connection...\n");
|
||||
rpc_->clearAbort();
|
||||
if (sendStopCommandSafely(*rpc_, "existing connection stop"))
|
||||
stop_sent = true;
|
||||
// Credentials for the stop: our own node's conf (and, if those are stale, our last working
|
||||
// connection, then the .cookie); a node we don't own, the exact credentials we connected with.
|
||||
std::vector<rpc::ConnectionConfig> creds;
|
||||
const rpc::ConnectionConfig detected = rpc::Connection::autoDetectConfig();
|
||||
const bool haveSaved = !saved_config_.rpcuser.empty() || !saved_config_.rpcpassword.empty();
|
||||
if (owned) {
|
||||
creds.push_back(detected);
|
||||
if (haveSaved && (saved_config_.rpcuser != detected.rpcuser ||
|
||||
saved_config_.rpcpassword != detected.rpcpassword || saved_config_.port != detected.port))
|
||||
creds.push_back(saved_config_);
|
||||
rpc::ConnectionConfig cookie;
|
||||
if (rpc::Connection::buildCookieAuthConfig(detected, cookie)) creds.push_back(cookie);
|
||||
} else {
|
||||
creds.push_back(haveSaved ? saved_config_ : detected);
|
||||
}
|
||||
|
||||
// If the main connection wasn't established (e.g. daemon was still
|
||||
// starting up when user closed the window), create a temporary
|
||||
// RPC connection just to send the stop command.
|
||||
if (!stop_sent) {
|
||||
DEBUG_LOGF("Main RPC not connected — creating temporary connection for stop...\n");
|
||||
auto config = rpc::Connection::autoDetectConfig();
|
||||
if (!config.rpcuser.empty() && !config.rpcpassword.empty()) {
|
||||
auto tmp_rpc = std::make_unique<rpc::RPCClient>();
|
||||
// Use a short timeout — if daemon isn't listening yet, don't block
|
||||
if (tmp_rpc->connect(config.host, config.port,
|
||||
config.rpcuser, config.rpcpassword,
|
||||
config.use_tls)) {
|
||||
DEBUG_LOGF("Temporary RPC connected, sending stop...\n");
|
||||
if (sendStopCommandSafely(*tmp_rpc, "Temporary daemon stop")) {
|
||||
stop_sent = true;
|
||||
}
|
||||
tmp_rpc->disconnect();
|
||||
} else {
|
||||
DEBUG_LOGF("Could not establish temporary RPC connection\n");
|
||||
}
|
||||
} else {
|
||||
DEBUG_LOGF("No RPC credentials available (DRAGONX.conf missing?)\n");
|
||||
}
|
||||
}
|
||||
|
||||
if (stop_sent) {
|
||||
DEBUG_LOGF("Waiting for daemon to flush block index and shut down...\n");
|
||||
shutdown_status_ = "Waiting for daemon to flush block index...";
|
||||
// Give the daemon time to flush LevelDB to disk before we
|
||||
// escalate to SIGTERM. On macOS/APFS, LevelDB compaction +
|
||||
// fsync can take 15-20s on a large chain. The stop() method
|
||||
// will wait this long for a *natural* exit (via the RPC stop
|
||||
// we already sent) before falling back to SIGTERM.
|
||||
}
|
||||
|
||||
shutdown_status_ = "Waiting for dragonxd process to exit...";
|
||||
// 20s grace period for the RPC "stop" to complete (LevelDB flush).
|
||||
// Only after that does stop() escalate to SIGTERM, then SIGKILL.
|
||||
daemon_controller_->stop(20000);
|
||||
|
||||
// EXTERNAL / adopted node during app shutdown: we hold no process handle, so the stop() above returned
|
||||
// immediately (it can only wait on a node WE spawned). But the user turned on "Stop external daemon",
|
||||
// so keep the window on the shutdown screen and poll until the node is actually gone — surfacing a live
|
||||
// status so they can SEE it stop — rather than closing while it's still flushing. Bounded ~120s (a
|
||||
// graceful full-node shutdown can flush LevelDB for 60-90s). Scoped to real shutdown; the shutdown
|
||||
// screen's Force Quit stays available and flips shutdown_complete_, which breaks us out at once.
|
||||
if (stop_sent && !owned && shutting_down_) {
|
||||
auto stillUp = []() {
|
||||
return daemon::EmbeddedDaemon::isRpcPortInUse() || daemon::EmbeddedDaemon::isDaemonProcessRunning();
|
||||
};
|
||||
// Set the phase text ONCE, then just poll — the shutdown screen already renders a live "N seconds"
|
||||
// elapsed counter on the UI thread, so the user still sees time passing. (shutdown_status_ is now a
|
||||
// GuardedStatus, so per-iteration writes would be race-safe; the single write is just a UX choice.)
|
||||
shutdown_status_ = "Waiting for the external node to stop...";
|
||||
for (int i = 0; i < 1200 && stillUp() && !shutdown_complete_; ++i)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
shutdown_status_ = stillUp() ? "External node still stopping — closing anyway..."
|
||||
: "External node stopped";
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: external node %s\n", stillUp() ? "still up (timed out)" : "confirmed stopped");
|
||||
daemon::DaemonController::StopPolicy policy;
|
||||
policy.owned = owned;
|
||||
#ifndef _WIN32
|
||||
policy.gracefulSignal = owned; // dragonxd treats SIGTERM like stop; Windows has no such channel
|
||||
#endif
|
||||
switch (mode) {
|
||||
case StopMode::AppShutdown:
|
||||
policy.userChoices = true;
|
||||
policy.maxWaitSec = owned ? 0.0 : 120.0; // a node we don't own: watch it go for up to 2 min
|
||||
break;
|
||||
case StopMode::Background:
|
||||
policy.maxWaitSec = 120.0;
|
||||
policy.abandonOnShutdown = true;
|
||||
stop_user_choice_ = 0;
|
||||
break;
|
||||
case StopMode::Fallback:
|
||||
policy.maxWaitSec = 20.0;
|
||||
stop_user_choice_ = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s node, %s\n", owned ? "our own" : "an external",
|
||||
mode == StopMode::AppShutdown ? "quitting (no time limit; the user may leave it running)"
|
||||
: mode == StopMode::Background ? "background (up to 120s, never killed)"
|
||||
: "fallback (up to 20s, never killed)");
|
||||
stop_kill_offered_ = false;
|
||||
stop_session_active_ = true;
|
||||
daemon::StopOutcome outcome;
|
||||
{
|
||||
AppStopRuntime runtime(*this, owned ? d : nullptr, std::move(creds));
|
||||
AppStopTaskContext task(*this, policy.abandonOnShutdown);
|
||||
outcome = daemon::DaemonController::executeDaemonStop(policy, runtime, task);
|
||||
}
|
||||
stop_session_active_ = false;
|
||||
stop_kill_offered_ = false;
|
||||
// Still running after a background wait: keep watching it (its exit is booked as a clean stop). Left
|
||||
// running on purpose ("Leave running"), exited or killed: let go of it.
|
||||
if (owned) d->endStop(/*keepWatching=*/outcome == daemon::StopOutcome::StillRunning);
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s\n",
|
||||
outcome == daemon::StopOutcome::Exited ? "the node exited"
|
||||
: outcome == daemon::StopOutcome::Detached ? "left running at the user's request"
|
||||
: outcome == daemon::StopOutcome::Killed ? "killed at the user's request"
|
||||
: "still running (stopped waiting; never killed)");
|
||||
return outcome;
|
||||
}
|
||||
|
||||
bool App::isEmbeddedDaemonRunning() const
|
||||
@@ -5493,10 +5619,10 @@ bool App::stopDaemonForWalletSwitch()
|
||||
// not start the node. Gate on the handle instead.
|
||||
const bool owned = daemon_controller_ && isEmbeddedDaemonRunning();
|
||||
if (owned) {
|
||||
// We spawned it — stopEmbeddedDaemon() BLOCKS for full process exit (handle wait + SIGTERM/SIGKILL
|
||||
// escalation), so the datadir lock + RPC port are released by the time it returns; a short port
|
||||
// check just confirms.
|
||||
stopEmbeddedDaemon();
|
||||
// We spawned it — stopEmbeddedDaemon() waits for the process to exit (up to 120s, never killing
|
||||
// it), so on Exited the datadir lock + RPC port are released; a short port check just confirms.
|
||||
// Still running means the switch must not go ahead: the old node holds the wallet and the lock.
|
||||
if (stopEmbeddedDaemon() != daemon::StopOutcome::Exited) return shutting_down_;
|
||||
for (int i = 0; i < 200 && daemon::EmbeddedDaemon::isRpcPortInUse() && !shutting_down_; ++i)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
return shutting_down_ || !daemon::EmbeddedDaemon::isRpcPortInUse();
|
||||
@@ -5808,6 +5934,8 @@ void App::beginShutdown()
|
||||
shutting_down_ = true;
|
||||
quit_requested_ = true;
|
||||
shutdown_timer_ = 0.0f;
|
||||
stop_user_choice_ = 0;
|
||||
stop_kill_offered_ = false;
|
||||
shutdown_start_time_ = std::chrono::steady_clock::now();
|
||||
async_tasks_.cancelAll();
|
||||
// The seed-migration adopt task drives daemon stop/start and swaps wallet.dat; let it finish
|
||||
@@ -5883,8 +6011,9 @@ void App::beginShutdown()
|
||||
shutdown_thread_ = std::thread([this]() {
|
||||
DEBUG_LOGF("shutdown thread: calling stopEmbeddedDaemon()\n");
|
||||
shutdown_status_ = TR("appx_sending_stop_command_to_daemon");
|
||||
// Send RPC stop command
|
||||
stopEmbeddedDaemon();
|
||||
// Deliver stop and wait for the node to exit: no time limit, never killed unasked. The shutdown
|
||||
// screen offers "Leave running" (and "Kill" if the node goes idle), which end the wait at once.
|
||||
stopEmbeddedDaemon(StopMode::AppShutdown);
|
||||
|
||||
DEBUG_LOGF("shutdown thread: daemon stopped, disconnecting RPC\n");
|
||||
shutdown_status_ = TR("appx_cleaning_up");
|
||||
@@ -6055,17 +6184,10 @@ void App::renderShutdownScreen()
|
||||
};
|
||||
shutdown_timer_ += ImGui::GetIO().DeltaTime;
|
||||
|
||||
// Track how long the status text has been unchanged. A normal shutdown keeps updating the status
|
||||
// ("Stopping pool miner…" -> "Flushing…" -> "Cleaning up…"); a genuine hang stalls it. Offer Force
|
||||
// Quit only once the status has STALLED (likely hung), not on a bare timer, with a hard ceiling so
|
||||
// it's never impossible to escape.
|
||||
static std::string s_lastShutStatus;
|
||||
static float s_shutStallTimer = 0.0f;
|
||||
const std::string curShut = shutdown_status_.get(); // one consistent snapshot per frame (M-05)
|
||||
if (curShut != s_lastShutStatus) { s_lastShutStatus = curShut; s_shutStallTimer = 0.0f; }
|
||||
else s_shutStallTimer += ImGui::GetIO().DeltaTime;
|
||||
const bool shutdownStalled = s_shutStallTimer >= 8.0f;
|
||||
const bool allowForceQuit = shutdownStalled || shutdown_timer_ >= 20.0f;
|
||||
// After a few seconds, offer to close the wallet and leave the node running: stopping can take as long
|
||||
// as the node needs (a witness rebuild can hold it for an hour), and it is never killed unasked.
|
||||
const bool allowChoices = shutdown_timer_ >= 3.0f;
|
||||
|
||||
// Use the main viewport so the overlay covers the primary window
|
||||
ImGuiViewport* vp = ImGui::GetMainViewport();
|
||||
@@ -6086,9 +6208,8 @@ void App::renderShutdownScreen()
|
||||
ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize |
|
||||
ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoScrollbar |
|
||||
ImGuiWindowFlags_NoSavedSettings;
|
||||
// Allow input only once Force Quit is offered (status stalled / hard ceiling), so the screen stays
|
||||
// click-through while a normal shutdown is progressing.
|
||||
if (!allowForceQuit)
|
||||
// Allow input only once the choices are offered, so the screen stays click-through for a quick exit.
|
||||
if (!allowChoices)
|
||||
shutdownFlags |= ImGuiWindowFlags_NoInputs | ImGuiWindowFlags_NoNav;
|
||||
ImGui::Begin("##ShutdownOverlay", nullptr, shutdownFlags);
|
||||
|
||||
@@ -6189,51 +6310,64 @@ void App::renderShutdownScreen()
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
// 4b. Force Quit button — appears once the shutdown appears stalled (or a hard-ceiling backstop)
|
||||
// 4b. "Leave running" (always, after a few seconds) and "Kill" (only once the node looks stuck)
|
||||
// -------------------------------------------------------------------
|
||||
if (allowForceQuit) {
|
||||
if (allowChoices) {
|
||||
ImGui::Spacing();
|
||||
ImGui::Spacing();
|
||||
// State-aware caution: while the status is a daemon flush/exit step, force-quitting risks the
|
||||
// chainstate; say so instead of a bare button.
|
||||
if (shutdownStalled && !curShut.empty()) {
|
||||
std::string stalledMsg = std::string(TR("appx_still_status_prefix")) + curShut +
|
||||
TR("appx_still_status_suffix");
|
||||
ImVec2 ms = ImGui::CalcTextSize(stalledMsg.c_str());
|
||||
ImGui::SetCursorPosX(cx - ms.x * 0.5f);
|
||||
ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(Warning()));
|
||||
ImGui::TextUnformatted(stalledMsg.c_str());
|
||||
ImGui::PopStyleColor();
|
||||
ImGui::Spacing();
|
||||
}
|
||||
const char* forceLabel = TR("force_quit");
|
||||
ImVec2 btnSize(ImGui::CalcTextSize(forceLabel).x + 32.0f * dp, 0);
|
||||
ImGui::SetCursorPosX(cx - btnSize.x * 0.5f);
|
||||
ImGui::PushStyleColor(ImGuiCol_Button, ImVec4(0.6f, 0.15f, 0.15f, 0.9f));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.75f, 0.2f, 0.2f, 1.0f));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImVec4(0.5f, 0.1f, 0.1f, 1.0f));
|
||||
if (ui::material::TactileButton(forceLabel, btnSize)) {
|
||||
force_quit_confirm_ = true;
|
||||
const char* leaveLabel = TR("shutdown_leave_running");
|
||||
ImVec2 leaveSize(ImGui::CalcTextSize(leaveLabel).x + 32.0f * dp, 0);
|
||||
ImGui::SetCursorPosX(cx - leaveSize.x * 0.5f);
|
||||
if (ui::material::TactileButton(leaveLabel, leaveSize)) {
|
||||
DEBUG_LOGF("Shutdown: user chose to leave the node running after %.0fs\n", shutdown_timer_);
|
||||
chooseStopAction(daemon::StopUserChoice::Detach);
|
||||
// No stop session (yet): nothing to wait for, so close now. A session that starts later sees
|
||||
// the choice and returns at once.
|
||||
if (!stop_session_active_) shutdown_complete_ = true;
|
||||
}
|
||||
if (ImGui::IsItemHovered()) {
|
||||
ui::material::Tooltip("%s", TR("force_quit_warning"));
|
||||
ui::material::Tooltip("%s", TR("shutdown_leave_running_tip"));
|
||||
}
|
||||
|
||||
if (stop_kill_offered_) {
|
||||
ImGui::Spacing();
|
||||
char idleMsg[160];
|
||||
snprintf(idleMsg, sizeof(idleMsg), TR("shutdown_stop_idle"), stop_idle_minutes_.load());
|
||||
ImVec2 ms = ImGui::CalcTextSize(idleMsg);
|
||||
ImGui::SetCursorPosX(cx - ms.x * 0.5f);
|
||||
ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(Warning()));
|
||||
ImGui::TextUnformatted(idleMsg);
|
||||
ImGui::PopStyleColor();
|
||||
ImGui::Spacing();
|
||||
const char* killLabel = TR("shutdown_kill_node");
|
||||
ImVec2 killSize(ImGui::CalcTextSize(killLabel).x + 32.0f * dp, 0);
|
||||
ImGui::SetCursorPosX(cx - killSize.x * 0.5f);
|
||||
ImGui::PushStyleColor(ImGuiCol_Button, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 210)));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImGui::ColorConvertU32ToFloat4(Error()));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 160)));
|
||||
if (ui::material::TactileButton(killLabel, killSize)) {
|
||||
stop_kill_confirm_ = true;
|
||||
}
|
||||
ImGui::PopStyleColor(3);
|
||||
if (ImGui::IsItemHovered()) {
|
||||
ui::material::Tooltip("%s", TR("shutdown_kill_node_tip"));
|
||||
}
|
||||
}
|
||||
ImGui::PopStyleColor(3);
|
||||
}
|
||||
|
||||
// Force Quit confirmation popup
|
||||
if (force_quit_confirm_) {
|
||||
ImGui::OpenPopup("##ForceQuitConfirm");
|
||||
force_quit_confirm_ = false;
|
||||
// Kill confirmation
|
||||
if (stop_kill_confirm_) {
|
||||
ImGui::OpenPopup("##KillNodeConfirm");
|
||||
stop_kill_confirm_ = false;
|
||||
}
|
||||
ImVec2 popupCenter(wp.x + vp_size.x * 0.5f, wp.y + vp_size.y * 0.5f);
|
||||
ImGui::SetNextWindowPos(popupCenter, ImGuiCond_Appearing, ImVec2(0.5f, 0.5f));
|
||||
if (ImGui::BeginPopupModal("##ForceQuitConfirm", nullptr, ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoTitleBar)) {
|
||||
if (ImGui::BeginPopupModal("##KillNodeConfirm", nullptr, ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoTitleBar)) {
|
||||
ImGui::PushFont(Type().subtitle1());
|
||||
ImGui::TextUnformatted(TR("force_quit_confirm_title"));
|
||||
ImGui::TextUnformatted(TR("shutdown_kill_confirm_title"));
|
||||
ImGui::PopFont();
|
||||
ImGui::Spacing();
|
||||
ImGui::TextUnformatted(TR("force_quit_confirm_msg"));
|
||||
ImGui::TextUnformatted(TR("shutdown_kill_confirm_msg"));
|
||||
ImGui::Spacing();
|
||||
ImGui::Spacing();
|
||||
|
||||
@@ -6249,9 +6383,9 @@ void App::renderShutdownScreen()
|
||||
ImGui::PushStyleColor(ImGuiCol_Button, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 210)));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImGui::ColorConvertU32ToFloat4(Error()));
|
||||
ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 160)));
|
||||
if (ui::material::TactileButton(TR("force_quit_yes"), ImVec2(btnW, 0))) {
|
||||
DEBUG_LOGF("Force quit confirmed by user after %.0fs\n", shutdown_timer_);
|
||||
shutdown_complete_ = true;
|
||||
if (ui::material::TactileButton(TR("shutdown_kill_confirm_yes"), ImVec2(btnW, 0))) {
|
||||
DEBUG_LOGF("Shutdown: user confirmed Kill after %.0fs\n", shutdown_timer_);
|
||||
chooseStopAction(daemon::StopUserChoice::Kill);
|
||||
ImGui::CloseCurrentPopup();
|
||||
}
|
||||
ImGui::PopStyleColor(3);
|
||||
@@ -6767,7 +6901,7 @@ void App::shutdown()
|
||||
settings_->save();
|
||||
}
|
||||
if (daemon_controller_) {
|
||||
stopEmbeddedDaemon();
|
||||
stopEmbeddedDaemon(StopMode::Fallback); // on the UI thread: bounded, never killed
|
||||
}
|
||||
if (rpc_) {
|
||||
rpc_->disconnect();
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
#include "data/wallet_index.h"
|
||||
#include "data/wallet_state.h"
|
||||
#include "rpc/connection.h"
|
||||
#include "daemon/stop_delivery.h"
|
||||
#include "services/network_refresh_service.h"
|
||||
#include "services/wallet_security_controller.h"
|
||||
#include "services/wallet_security_workflow.h"
|
||||
@@ -480,7 +481,13 @@ public:
|
||||
|
||||
// Embedded daemon control
|
||||
bool startEmbeddedDaemon();
|
||||
void stopEmbeddedDaemon();
|
||||
// How long stopEmbeddedDaemon() may wait, and who decides. It never kills the node on its own.
|
||||
enum class StopMode {
|
||||
AppShutdown, // quitting: no time limit; the shutdown screen offers "Leave running" (and "Kill" when stuck)
|
||||
Background, // restart/rescan/switch/...: up to 120s, and let go as soon as the app starts quitting
|
||||
Fallback, // the synchronous shutdown fallback on the UI thread: up to 20s
|
||||
};
|
||||
daemon::StopOutcome stopEmbeddedDaemon(StopMode mode = StopMode::Background);
|
||||
// Stop the node specifically for a wallet switch, which MUST free the RPC port to relaunch on
|
||||
// -wallet=<name>. Unlike stopEmbeddedDaemon() (whose DisconnectOnly policy leaves an adopted/external
|
||||
// daemon running), this sends a graceful RPC "stop" — using our own connection creds, so only our
|
||||
@@ -632,6 +639,8 @@ public:
|
||||
|
||||
private:
|
||||
friend class AppDaemonLifecycleRuntime;
|
||||
friend class AppStopRuntime;
|
||||
friend class AppStopTaskContext;
|
||||
friend class AppDaemonLifecycleTaskContext;
|
||||
|
||||
// Global keyboard-shortcut handling, dispatched once per frame from update().
|
||||
@@ -964,9 +973,19 @@ private:
|
||||
bool shutdown_confirmed_ = false; // user chose to proceed — bypass the guard on re-entry
|
||||
bool shutdown_keep_daemon_override_ = false; // user chose "keep node running" for this shutdown only
|
||||
float shutdown_timer_ = 0.0f;
|
||||
bool force_quit_confirm_ = false;
|
||||
std::chrono::steady_clock::time_point shutdown_start_time_;
|
||||
|
||||
// One stop session at a time (stopEmbeddedDaemon), and the shutdown screen's view of it.
|
||||
std::mutex stop_mutex_;
|
||||
std::atomic<bool> stop_session_active_{false};
|
||||
std::atomic<int> stop_user_choice_{0}; // daemon::StopUserChoice, set by the shutdown screen
|
||||
std::atomic<bool> stop_kill_offered_{false}; // the node looks stuck: the screen may offer "Kill"
|
||||
std::atomic<int> stop_idle_minutes_{0};
|
||||
std::mutex stop_client_mutex_;
|
||||
rpc::RPCClient* active_stop_client_ = nullptr; // aborted on "Leave running" so a pending attempt ends at once
|
||||
bool stop_kill_confirm_ = false;
|
||||
void chooseStopAction(daemon::StopUserChoice choice);
|
||||
|
||||
// Daemon restart (e.g. after changing debug log categories)
|
||||
std::atomic<bool> daemon_restarting_{false};
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "embedded_daemon.h"
|
||||
#include "stop_delivery.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
@@ -234,6 +236,215 @@ public:
|
||||
ShutdownDecision shutdownDecision(bool keepDaemonRunning,
|
||||
bool stopExternalDaemon) const;
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Stopping the node: deliver "stop" until it lands, then wait for the process to exit. Never kill it
|
||||
// on our own: a node that is busy (a witness rebuild can hold it for an hour) or flushing would lose
|
||||
// everything since its last save and replay it on the next start. Only an explicit user "Kill",
|
||||
// offered once the node has gone idle, ends it by force; "Leave running" lets the wallet close while
|
||||
// the node finishes on its own.
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
using StopUserChoice = daemon::StopUserChoice;
|
||||
using StopOutcome = daemon::StopOutcome;
|
||||
|
||||
struct StopPolicy {
|
||||
bool owned = true; // we launched it: watch the process, signal it, offer Kill
|
||||
bool gracefulSignal = false; // POSIX: a group SIGTERM is a stop channel (dragonxd treats it as stop)
|
||||
double maxWaitSec = 0.0; // stop waiting (never kill) after this long; 0 = no limit
|
||||
bool userChoices = false; // the shutdown screen offers "Leave running" / "Kill"
|
||||
bool abandonOnShutdown = false; // stop waiting once the app starts shutting down (background callers)
|
||||
double idleOfferSec = 600.0; // delivered + no CPU/I-O progress for this long: offer Kill
|
||||
};
|
||||
|
||||
// What the stop loop reports for display.
|
||||
struct StopStatus {
|
||||
bool delivered = false; // stop accepted (or the node is shutting down, or was signalled)
|
||||
bool heard = false; // the node has the request (possibly still queued)
|
||||
bool signalled = false; // a graceful signal (POSIX SIGTERM) went out
|
||||
double elapsedSec = 0.0;
|
||||
double cpuPercent = -1.0; // < 0: unknown
|
||||
double idleSec = 0.0;
|
||||
bool killOffered = false;
|
||||
bool rebuildActive = false;
|
||||
StopSendResult lastSend = StopSendResult::Other;
|
||||
int attempts = 0;
|
||||
};
|
||||
|
||||
class StopRuntime {
|
||||
public:
|
||||
virtual ~StopRuntime() = default;
|
||||
virtual double nowSec() = 0;
|
||||
virtual bool processAlive() = 0;
|
||||
virtual StopSendResult trySendStop() = 0;
|
||||
virtual bool nextCredentials() = 0; // switch to the next stop credentials; false when none left
|
||||
virtual bool sendGracefulSignal() = 0; // POSIX group SIGTERM; false where there is none (Windows)
|
||||
virtual double cpuSeconds() = 0; // cumulative CPU of the node; < 0 if unknown
|
||||
virtual double ioBytes() = 0; // cumulative I/O bytes of the node; < 0 if unknown
|
||||
virtual bool rebuildActive() = 0; // a witness rebuild is running: never offer Kill
|
||||
virtual StopUserChoice userChoice() = 0;
|
||||
virtual void offerForceKill(bool offered) = 0;
|
||||
virtual void forceKill() = 0;
|
||||
virtual void setStatus(const StopStatus& status) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* "Making progress" = >= 0.05 cores of CPU or >= 1 MB of I/O over the last >= 60s. A flush that
|
||||
* fsyncs a big wallet is mostly I/O at low CPU; a rebuild is mostly CPU. Idle time counts from the
|
||||
* first sample. With no readings at all (unknown), it never reports idle, so Kill is never offered.
|
||||
*/
|
||||
class StopProgressTracker {
|
||||
public:
|
||||
static constexpr double kWindowSec = 60.0;
|
||||
static constexpr double kMinCores = 0.05;
|
||||
static constexpr double kMinIoBytes = 1024.0 * 1024.0;
|
||||
|
||||
void add(double t, double cpuSec, double ioBytes)
|
||||
{
|
||||
if (cpuSec >= 0 || ioBytes >= 0) measurable_ = true;
|
||||
if (samples_.empty()) lastProgressAt_ = t;
|
||||
samples_.push_back({t, cpuSec, ioBytes});
|
||||
// Keep the newest sample that is at least a full window old as the comparison base.
|
||||
while (samples_.size() > 2 && samples_[1].t <= t - kWindowSec) samples_.pop_front();
|
||||
const Sample& base = samples_.front();
|
||||
const double span = t - base.t;
|
||||
if (span >= kWindowSec) {
|
||||
const bool cpuMoved = cpuSec >= 0 && base.cpu >= 0 && (cpuSec - base.cpu) / span >= kMinCores;
|
||||
const bool ioMoved = ioBytes >= 0 && base.io >= 0 && (ioBytes - base.io) >= kMinIoBytes;
|
||||
if (cpuMoved || ioMoved) lastProgressAt_ = t;
|
||||
}
|
||||
}
|
||||
double idleSec(double t) const { return measurable_ ? std::max(0.0, t - lastProgressAt_) : 0.0; }
|
||||
// CPU over the most recent sample interval (100 = one core), for display; < 0 unknown.
|
||||
double cpuPercent() const
|
||||
{
|
||||
if (samples_.size() < 2) return -1.0;
|
||||
const Sample& a = samples_[samples_.size() - 2];
|
||||
const Sample& b = samples_.back();
|
||||
if (a.cpu < 0 || b.cpu < 0 || b.t <= a.t) return -1.0;
|
||||
return std::max(0.0, (b.cpu - a.cpu) / (b.t - a.t) * 100.0);
|
||||
}
|
||||
|
||||
private:
|
||||
struct Sample { double t, cpu, io; };
|
||||
std::deque<Sample> samples_;
|
||||
double lastProgressAt_ = 0.0;
|
||||
bool measurable_ = false;
|
||||
};
|
||||
|
||||
enum class StopWaitDecision { Done, Wait, GiveUp, Detach, OfferForceKill, ForceKill };
|
||||
|
||||
struct StopWaitSample {
|
||||
bool processAlive = true;
|
||||
bool delivered = false;
|
||||
double elapsedSec = 0.0;
|
||||
double idleSec = 0.0;
|
||||
bool rebuildActive = false;
|
||||
StopUserChoice choice = StopUserChoice::None;
|
||||
bool killOffered = false; // the Kill offer is on screen right now
|
||||
bool shuttingDown = false; // the app began shutting down / the task was cancelled
|
||||
};
|
||||
|
||||
static inline StopWaitDecision evaluateStopWait(const StopPolicy& p, const StopWaitSample& s)
|
||||
{
|
||||
if (!s.processAlive) return StopWaitDecision::Done;
|
||||
if (p.userChoices && s.choice == StopUserChoice::Detach) return StopWaitDecision::Detach;
|
||||
if (p.userChoices && p.owned && s.choice == StopUserChoice::Kill && s.killOffered)
|
||||
return StopWaitDecision::ForceKill;
|
||||
if (p.abandonOnShutdown && s.shuttingDown) return StopWaitDecision::GiveUp;
|
||||
if (p.maxWaitSec > 0 && s.elapsedSec >= p.maxWaitSec) return StopWaitDecision::GiveUp;
|
||||
if (p.userChoices && p.owned && s.delivered && !s.rebuildActive && s.idleSec >= p.idleOfferSec)
|
||||
return StopWaitDecision::OfferForceKill;
|
||||
return StopWaitDecision::Wait;
|
||||
}
|
||||
|
||||
static constexpr int kStopTickMs = 250;
|
||||
|
||||
static inline StopOutcome executeDaemonStop(const StopPolicy& policy, StopRuntime& rt,
|
||||
LifecycleTaskContext& task)
|
||||
{
|
||||
if (!rt.processAlive()) return StopOutcome::Exited; // already gone: not a single RPC call
|
||||
|
||||
const double t0 = rt.nowSec();
|
||||
StopDeliveryPolicy delivery(policy.owned);
|
||||
StopProgressTracker progress;
|
||||
bool signalled = false;
|
||||
bool offered = false;
|
||||
auto withdrawOffer = [&] {
|
||||
if (offered) { offered = false; rt.offerForceKill(false); }
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
double now = rt.nowSec() - t0;
|
||||
if (delivery.shouldSend(now)) {
|
||||
const StopSendResult r = delivery.record(rt.trySendStop(), now);
|
||||
if (r == StopSendResult::Auth && policy.owned) {
|
||||
// Our own node refusing our credentials means they're stale (e.g. a regenerated
|
||||
// .cookie): try the next source; out of sources, the graceful signal below remains.
|
||||
if (rt.nextCredentials()) delivery.retryNow(rt.nowSec() - t0);
|
||||
else delivery.giveUp();
|
||||
}
|
||||
// POSIX: a group SIGTERM bypasses the RPC queue altogether (dragonxd's handler sets the same
|
||||
// flag as stop). Send it as soon as an attempt shows stop didn't get through.
|
||||
if (policy.gracefulSignal && !signalled && !delivery.delivered() &&
|
||||
r != StopSendResult::Queued && rt.sendGracefulSignal()) {
|
||||
signalled = true;
|
||||
delivery.markDelivered();
|
||||
}
|
||||
}
|
||||
if (!rt.processAlive()) {
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
}
|
||||
|
||||
now = rt.nowSec() - t0;
|
||||
progress.add(now, rt.cpuSeconds(), rt.ioBytes());
|
||||
StopWaitSample s;
|
||||
s.delivered = delivery.delivered();
|
||||
s.elapsedSec = now;
|
||||
s.idleSec = progress.idleSec(now);
|
||||
s.rebuildActive = rt.rebuildActive();
|
||||
s.choice = rt.userChoice();
|
||||
s.killOffered = offered;
|
||||
s.shuttingDown = task.cancelled() || task.shuttingDown();
|
||||
|
||||
switch (evaluateStopWait(policy, s)) {
|
||||
case StopWaitDecision::Done:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Exited;
|
||||
case StopWaitDecision::Detach:
|
||||
withdrawOffer();
|
||||
return StopOutcome::Detached;
|
||||
case StopWaitDecision::ForceKill:
|
||||
withdrawOffer();
|
||||
rt.forceKill();
|
||||
return StopOutcome::Killed;
|
||||
case StopWaitDecision::GiveUp:
|
||||
withdrawOffer();
|
||||
return StopOutcome::StillRunning;
|
||||
case StopWaitDecision::OfferForceKill:
|
||||
if (!offered) { offered = true; rt.offerForceKill(true); }
|
||||
break;
|
||||
case StopWaitDecision::Wait:
|
||||
withdrawOffer(); // progress resumed (or a rebuild started): the node isn't stuck after all
|
||||
break;
|
||||
}
|
||||
|
||||
StopStatus st;
|
||||
st.delivered = delivery.delivered();
|
||||
st.heard = delivery.heard();
|
||||
st.signalled = signalled;
|
||||
st.elapsedSec = now;
|
||||
st.cpuPercent = progress.cpuPercent();
|
||||
st.idleSec = s.idleSec;
|
||||
st.killOffered = offered;
|
||||
st.rebuildActive = s.rebuildActive;
|
||||
st.lastSend = delivery.last();
|
||||
st.attempts = delivery.attempts();
|
||||
rt.setStatus(st);
|
||||
task.sleepForMs(kStopTickMs);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<EmbeddedDaemon> daemon_;
|
||||
};
|
||||
|
||||
@@ -948,6 +948,78 @@ bool EmbeddedDaemon::stop(int wait_ms)
|
||||
return exitedOnItsOwn;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::beginStop()
|
||||
{
|
||||
stop_requested_ = true;
|
||||
should_stop_ = true;
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
if (!isRunning()) return false; // handle-based: our own process only
|
||||
setState(State::Stopping, "Stopping dragonxd...");
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::pollOwnedProcess()
|
||||
{
|
||||
drainOutput();
|
||||
return isRunning();
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::signalGracefulStop()
|
||||
{
|
||||
return false; // no graceful channel on Windows: the GUI launches dragonxd without a console
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::forceKillOwnedProcess()
|
||||
{
|
||||
if (process_handle_ == nullptr || !isRunning()) return;
|
||||
DEBUG_LOGF("Killing dragonxd at the user's request (TerminateProcess on our own handle)\n");
|
||||
TerminateProcess(process_handle_, 1);
|
||||
WaitForSingleObject(process_handle_, 5000);
|
||||
drainOutput();
|
||||
}
|
||||
|
||||
EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const
|
||||
{
|
||||
ResourceUsage usage;
|
||||
if (process_handle_ == nullptr) return usage;
|
||||
FILETIME created, exited, kernel, user;
|
||||
if (GetProcessTimes(process_handle_, &created, &exited, &kernel, &user)) {
|
||||
auto ticks = [](const FILETIME& ft) {
|
||||
return (static_cast<unsigned long long>(ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
|
||||
};
|
||||
usage.cpuSeconds = static_cast<double>(ticks(kernel) + ticks(user)) / 1e7; // 100ns units
|
||||
}
|
||||
IO_COUNTERS io;
|
||||
if (GetProcessIoCounters(process_handle_, &io))
|
||||
usage.ioBytes = static_cast<double>(io.ReadTransferCount + io.WriteTransferCount);
|
||||
return usage;
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::endStop(bool keepWatching)
|
||||
{
|
||||
if (keepWatching && isRunning()) {
|
||||
// Still running and still ours: watch it again. Its exit is booked as a clean stop.
|
||||
should_stop_ = false;
|
||||
monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
|
||||
return;
|
||||
}
|
||||
const bool leftRunning = isRunning();
|
||||
if (leftRunning)
|
||||
DEBUG_LOGF("Leaving dragonxd running; it will finish shutting down on its own\n");
|
||||
if (process_handle_ != nullptr) {
|
||||
CloseHandle(process_handle_); // only our handle: the process carries on
|
||||
process_handle_ = nullptr;
|
||||
}
|
||||
if (stdout_read_ != nullptr) {
|
||||
CloseHandle(stdout_read_);
|
||||
stdout_read_ = nullptr;
|
||||
}
|
||||
drainOutput();
|
||||
setState(State::Stopped, leftRunning ? "dragonxd left running" : "dragonxd stopped");
|
||||
}
|
||||
|
||||
// Translate Windows NTSTATUS / exit codes to human-readable descriptions
|
||||
static std::string translateWindowsExitCode(DWORD code) {
|
||||
switch (code) {
|
||||
@@ -1396,6 +1468,125 @@ bool EmbeddedDaemon::stop(int wait_ms)
|
||||
return exitedOnItsOwn;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::beginStop()
|
||||
{
|
||||
stop_requested_ = true;
|
||||
should_stop_ = true;
|
||||
// The monitor sees should_stop_ within one 100ms tick. If the child exited in that window the
|
||||
// monitor reaped it (as a clean stop: stop_requested_ is set) and the pid is now 0.
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
if (process_pid_ <= 0) return false;
|
||||
setState(State::Stopping, "Stopping dragonxd...");
|
||||
return true;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::pollOwnedProcess()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return false;
|
||||
drainOutput();
|
||||
int status = 0;
|
||||
const pid_t r = waitpid(pid, &status, WNOHANG);
|
||||
if (r == 0 || (r < 0 && errno == EINTR)) return true;
|
||||
if (r == pid) DEBUG_LOGF("dragonxd exited (%s)\n", describeExitStatus(status).c_str());
|
||||
drainOutput();
|
||||
process_pid_ = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::signalGracefulStop()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return false;
|
||||
// The whole group: with a bash wrapper (embedded_daemon.cpp startProcess) a SIGTERM to the wrapper
|
||||
// alone would orphan dragonxd.
|
||||
DEBUG_LOGF("Sending SIGTERM to dragonxd's process group -%d (graceful stop)\n", pid);
|
||||
return kill(-pid, SIGTERM) == 0;
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::forceKillOwnedProcess()
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid <= 0) return;
|
||||
DEBUG_LOGF("Killing dragonxd (group -%d) at the user's request\n", pid);
|
||||
kill(-pid, SIGKILL);
|
||||
int status = 0;
|
||||
while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {}
|
||||
drainOutput();
|
||||
process_pid_ = 0;
|
||||
}
|
||||
|
||||
EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const
|
||||
{
|
||||
ResourceUsage usage;
|
||||
#ifdef __APPLE__
|
||||
return usage; // no sampler yet (would need proc_pid_rusage): never "idle", so Kill is never offered
|
||||
#else
|
||||
const pid_t pgid = process_pid_;
|
||||
if (pgid <= 0) return usage;
|
||||
// The tracked pid may be a bash wrapper that itself uses ~0 CPU: sum the whole process group.
|
||||
unsigned long long ticks = 0;
|
||||
double io = 0.0;
|
||||
bool anyCpu = false, anyIo = false;
|
||||
std::error_code ec;
|
||||
fs::directory_iterator it("/proc", ec), end;
|
||||
for (; !ec && it != end; it.increment(ec)) {
|
||||
const std::string name = it->path().filename().string();
|
||||
if (name.empty() || name[0] < '0' || name[0] > '9') continue;
|
||||
std::ifstream statFile((it->path() / "stat").string());
|
||||
std::string line;
|
||||
long pgrp = 0;
|
||||
unsigned long long t = 0;
|
||||
if (!statFile || !std::getline(statFile, line) || !parseProcStatLine(line, pgrp, t) || pgrp != pgid)
|
||||
continue;
|
||||
ticks += t;
|
||||
anyCpu = true;
|
||||
std::ifstream ioFile((it->path() / "io").string());
|
||||
std::string key;
|
||||
unsigned long long value = 0;
|
||||
while (ioFile >> key >> value) {
|
||||
if (key == "read_bytes:" || key == "write_bytes:") {
|
||||
io += static_cast<double>(value);
|
||||
anyIo = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
const long hz = sysconf(_SC_CLK_TCK);
|
||||
if (anyCpu && hz > 0) usage.cpuSeconds = static_cast<double>(ticks) / static_cast<double>(hz);
|
||||
if (anyIo) usage.ioBytes = io;
|
||||
return usage;
|
||||
#endif
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::endStop(bool keepWatching)
|
||||
{
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid > 0 && keepWatching) {
|
||||
// Still running and still ours: watch it again. Its exit is booked as a clean stop.
|
||||
should_stop_ = false;
|
||||
monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
|
||||
return;
|
||||
}
|
||||
if (pid > 0) {
|
||||
// Leave it running. Close our end of its output pipe: its further writes fail with EPIPE, which
|
||||
// dragonxd ignores (it ignores SIGPIPE), instead of blocking once a pipe nobody drains is full.
|
||||
// A detached waiter reaps it when it exits, if this process is still around then.
|
||||
DEBUG_LOGF("Leaving dragonxd (pid %d) running; it will finish shutting down on its own\n", pid);
|
||||
std::thread([pid] {
|
||||
int st = 0;
|
||||
while (waitpid(pid, &st, 0) < 0 && errno == EINTR) {}
|
||||
}).detach();
|
||||
process_pid_ = 0;
|
||||
}
|
||||
if (stdout_fd_ >= 0) {
|
||||
close(stdout_fd_);
|
||||
stdout_fd_ = -1;
|
||||
}
|
||||
setState(State::Stopped, pid > 0 ? "dragonxd left running" : "dragonxd stopped");
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::monitorProcess()
|
||||
{
|
||||
char buffer[4096];
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -71,6 +72,47 @@ public:
|
||||
*/
|
||||
void markStopRequested() { stop_requested_ = true; }
|
||||
|
||||
// --- A stop session, driven by DaemonController::executeDaemonStop through the app's StopRuntime ---
|
||||
|
||||
/** Marks the stop as requested and stops the monitor, so the session is the only reaper.
|
||||
* @return true if a process this object launched is still alive; then finish with endStop(). */
|
||||
bool beginStop();
|
||||
/** Polls our own process (reaping it on POSIX) and drains its output; false once it has exited. */
|
||||
bool pollOwnedProcess();
|
||||
/** POSIX: SIGTERM our process group (dragonxd treats it like RPC stop). Windows has no such channel. */
|
||||
bool signalGracefulStop();
|
||||
/** Ends our own process by force (SIGKILL / TerminateProcess) and reaps it. Explicit user choice only. */
|
||||
void forceKillOwnedProcess();
|
||||
struct ResourceUsage {
|
||||
double cpuSeconds = -1.0; // cumulative; < 0 if unknown
|
||||
double ioBytes = -1.0; // cumulative storage I/O; < 0 if unknown
|
||||
};
|
||||
/** CPU and I/O used so far by our process (its whole process group on Linux). */
|
||||
ResourceUsage ownedResourceUsage() const;
|
||||
/** Finishes a stop session. The process is gone: release everything. Still running: keepWatching
|
||||
* resumes the monitor (its exit is still seen, and booked as a clean stop); otherwise let it go —
|
||||
* the user chose to leave it running. */
|
||||
void endStop(bool keepWatching);
|
||||
|
||||
/**
|
||||
* @brief Parse a /proc/<pid>/stat line: process group (field 5) and utime+stime ticks (fields 14+15)
|
||||
*
|
||||
* comm (field 2) is parenthesised and may itself contain spaces and ')', so fields are read from
|
||||
* after the LAST ')'.
|
||||
*/
|
||||
static bool parseProcStatLine(const std::string& line, long& pgrp, unsigned long long& cpuTicks)
|
||||
{
|
||||
const size_t close = line.rfind(')');
|
||||
if (close == std::string::npos) return false;
|
||||
unsigned long long utime = 0, stime = 0;
|
||||
// state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt cmajflt utime stime
|
||||
if (std::sscanf(line.c_str() + close + 1, " %*c %*d %ld %*d %*d %*d %*u %*u %*u %*u %*u %llu %llu",
|
||||
&pgrp, &utime, &stime) != 3)
|
||||
return false;
|
||||
cpuTicks = utime + stime;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Check if daemon process is running
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
#pragma once
|
||||
|
||||
// Delivering "stop" to dragonxd: what one attempt achieved, and when to send another. Pure and
|
||||
// header-only so the test target can drive it without the RPC client.
|
||||
//
|
||||
// Why retrying matters: dragonxd's HTTP server has 8 workers and a 16-deep queue. While a long
|
||||
// operation holds cs_main, the GUI's own abandoned polls can fill every slot, and each new request is
|
||||
// refused at once with "Work queue depth exceeded" until a slot frees. The old code made one or two
|
||||
// attempts, gated on a getinfo probe that needs cs_main, so stop was never delivered and the node was
|
||||
// then killed.
|
||||
|
||||
#include "rpc/rpc_failure.h"
|
||||
|
||||
namespace dragonx {
|
||||
namespace daemon {
|
||||
|
||||
// What the user picked on the shutdown screen while the node was stopping.
|
||||
enum class StopUserChoice { None, Detach, Kill };
|
||||
|
||||
// How a stop session ended. The wallet never kills the node unless the user chose Kill.
|
||||
enum class StopOutcome {
|
||||
Exited, // the process is gone
|
||||
StillRunning, // we stopped waiting (time budget, app shutdown, cancellation); never killed
|
||||
Detached, // the user chose to leave it running
|
||||
Killed, // the user chose Kill
|
||||
};
|
||||
|
||||
inline const char* stopOutcomeName(StopOutcome o)
|
||||
{
|
||||
switch (o) {
|
||||
case StopOutcome::Exited: return "Exited";
|
||||
case StopOutcome::StillRunning: return "StillRunning";
|
||||
case StopOutcome::Detached: return "Detached";
|
||||
case StopOutcome::Killed: return "Killed";
|
||||
}
|
||||
return "Exited";
|
||||
}
|
||||
|
||||
enum class StopSendResult {
|
||||
Accepted, // HTTP 200: the daemon ran stop
|
||||
Queued, // reply timeout: it reached the daemon and waits for a worker; it will run
|
||||
QueueFull, // "Work queue depth exceeded": refused before queueing (holds no slot)
|
||||
NotListening, // refused / connect timeout: RPC isn't bound (not up yet, or already unbound)
|
||||
ShuttingDown, // 503 / "Unhandled request": the daemon is already shutting down
|
||||
Auth, // 401: our credentials are wrong or stale
|
||||
Other,
|
||||
};
|
||||
|
||||
inline const char* stopSendResultName(StopSendResult r)
|
||||
{
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted: return "Accepted";
|
||||
case StopSendResult::Queued: return "Queued";
|
||||
case StopSendResult::QueueFull: return "QueueFull";
|
||||
case StopSendResult::NotListening: return "NotListening";
|
||||
case StopSendResult::ShuttingDown: return "ShuttingDown";
|
||||
case StopSendResult::Auth: return "Auth";
|
||||
case StopSendResult::Other: return "Other";
|
||||
}
|
||||
return "Other";
|
||||
}
|
||||
|
||||
// What a stop call's failure kind (RpcFailureKind::None = HTTP 200) means for delivery.
|
||||
inline StopSendResult stopSendResultFromKind(rpc::RpcFailureKind kind)
|
||||
{
|
||||
using K = rpc::RpcFailureKind;
|
||||
switch (kind) {
|
||||
case K::None: return StopSendResult::Accepted;
|
||||
case K::ReplyTimeout: return StopSendResult::Queued;
|
||||
case K::QueueFull: return StopSendResult::QueueFull;
|
||||
case K::NotListening:
|
||||
case K::ConnectTimeout: return StopSendResult::NotListening;
|
||||
case K::ShuttingDown:
|
||||
case K::Unhandled: return StopSendResult::ShuttingDown;
|
||||
case K::Auth: return StopSendResult::Auth;
|
||||
default: return StopSendResult::Other;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When to (re)send stop. Times are seconds on the caller's clock.
|
||||
*
|
||||
* - Accepted / ShuttingDown: delivered, never resend.
|
||||
* - Queued: the daemon has it. Resend at most every 30s: each resend takes another queue slot, and
|
||||
* concurrent stops race on the miner threads (stop's GenerateBitcoins(false)).
|
||||
* - QueueFull / NotListening / Other: not delivered, cheap to retry: again after 1.5s.
|
||||
* - NotListening AFTER the daemon had heard us means it unbound its listener to shut down: delivered.
|
||||
* - Auth: give up for a node we don't own. For our own node the caller may switch credentials and
|
||||
* retry at once (retryNow), or give up (giveUp).
|
||||
*/
|
||||
class StopDeliveryPolicy {
|
||||
public:
|
||||
static constexpr double kRetrySec = 1.5;
|
||||
static constexpr double kQueuedResendSec = 30.0;
|
||||
|
||||
explicit StopDeliveryPolicy(bool owned) : owned_(owned) {}
|
||||
|
||||
bool shouldSend(double now) const { return !finished_ && now >= nextSendAt_; }
|
||||
|
||||
// Records one attempt; returns it as interpreted (a refusal after we were heard reads as ShuttingDown).
|
||||
StopSendResult record(StopSendResult r, double now)
|
||||
{
|
||||
++attempts_;
|
||||
if (r == StopSendResult::NotListening && heard_) r = StopSendResult::ShuttingDown;
|
||||
switch (r) {
|
||||
case StopSendResult::Accepted:
|
||||
heard_ = true;
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::ShuttingDown:
|
||||
markDelivered();
|
||||
break;
|
||||
case StopSendResult::Queued:
|
||||
heard_ = true;
|
||||
nextSendAt_ = now + kQueuedResendSec;
|
||||
break;
|
||||
case StopSendResult::Auth:
|
||||
authRejected_ = true;
|
||||
if (!owned_) finished_ = true;
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
case StopSendResult::QueueFull:
|
||||
case StopSendResult::NotListening:
|
||||
case StopSendResult::Other:
|
||||
nextSendAt_ = now + kRetrySec;
|
||||
break;
|
||||
}
|
||||
last_ = r;
|
||||
return r;
|
||||
}
|
||||
|
||||
void markDelivered() { delivered_ = finished_ = true; } // also for a graceful signal (POSIX SIGTERM)
|
||||
void retryNow(double now) { nextSendAt_ = now; }
|
||||
void giveUp() { finished_ = true; }
|
||||
|
||||
bool delivered() const { return delivered_; } // the daemon is stopping (or was signalled)
|
||||
bool heard() const { return heard_; } // it has (or ran) our request
|
||||
bool finished() const { return finished_; } // no more attempts
|
||||
bool authRejected() const { return authRejected_; }
|
||||
int attempts() const { return attempts_; }
|
||||
StopSendResult last() const { return last_; }
|
||||
|
||||
private:
|
||||
bool owned_ = true;
|
||||
bool delivered_ = false;
|
||||
bool heard_ = false;
|
||||
bool finished_ = false;
|
||||
bool authRejected_ = false;
|
||||
int attempts_ = 0;
|
||||
double nextSendAt_ = 0.0;
|
||||
StopSendResult last_ = StopSendResult::Other;
|
||||
};
|
||||
|
||||
} // namespace daemon
|
||||
} // namespace dragonx
|
||||
+3
-4
@@ -2147,10 +2147,9 @@ int main(int argc, char* argv[])
|
||||
|
||||
// Watchdog: if cleanup takes too long the process lingers without a
|
||||
// window, showing up as a "Background Service" in Task Manager.
|
||||
// Force-exit after timeout — all critical state (settings, daemon
|
||||
// stop) was handled in beginShutdown().
|
||||
// Allow enough time for the daemon to shut down gracefully (up to
|
||||
// 10s in stop()) plus RPC disconnect overhead.
|
||||
// Force-exit after timeout. The daemon stop is already over by the time the render loop exits: the
|
||||
// node has exited, or the user chose to leave it running (it carries on as its own process), so this
|
||||
// only bounds the remaining thread joins and RPC disconnects — it never cuts a node's shutdown short.
|
||||
std::thread([]() {
|
||||
std::this_thread::sleep_for(std::chrono::seconds(15));
|
||||
fflush(stdout);
|
||||
|
||||
@@ -163,6 +163,17 @@ bool RPCClient::connect(const std::string& host, const std::string& port,
|
||||
bool useTls)
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lk(curl_mutex_);
|
||||
if (!open(host, port, user, password, useTls)) return false;
|
||||
return probeConnection(host);
|
||||
}
|
||||
|
||||
bool RPCClient::open(const std::string& host, const std::string& port,
|
||||
const std::string& user, const std::string& password,
|
||||
bool useTls)
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lk(curl_mutex_);
|
||||
connected_ = false; // only a successful probe (connect()) sets these
|
||||
warming_up_ = false;
|
||||
host_ = host;
|
||||
port_ = port;
|
||||
last_connect_info_ = json();
|
||||
@@ -218,7 +229,11 @@ bool RPCClient::connect(const std::string& host, const std::string& port,
|
||||
// build defaults. Harmless on the localhost http:// case; essential for a remote https daemon.
|
||||
curl_easy_setopt(impl_->curl, CURLOPT_SSL_VERIFYPEER, 1L);
|
||||
curl_easy_setopt(impl_->curl, CURLOPT_SSL_VERIFYHOST, 2L);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool RPCClient::probeConnection(const std::string& host)
|
||||
{
|
||||
// Test connection with getinfo. Use a SHORT timeout for the probe on localhost: a healthy
|
||||
// local daemon answers in milliseconds and a warming one returns -28 just as fast, so a long
|
||||
// hang means a wedged/loading occupant — no point blocking the full 30s before we retry and
|
||||
@@ -286,6 +301,17 @@ void RPCClient::logConnectFailure(const std::string& msg)
|
||||
DEBUG_LOGF("Connection failed %s (repeated %u times)\n", line.c_str(), connect_error_repeats_);
|
||||
}
|
||||
|
||||
RpcFailureKind RPCClient::trySendStop(long timeoutSec)
|
||||
{
|
||||
try {
|
||||
TraceScope trace("Daemon lifecycle / Stop command");
|
||||
call("stop", json::array(), timeoutSec);
|
||||
return RpcFailureKind::None;
|
||||
} catch (const std::exception& e) {
|
||||
return rpcFailureKindOf(e);
|
||||
}
|
||||
}
|
||||
|
||||
json RPCClient::getLastConnectInfo() const
|
||||
{
|
||||
std::lock_guard<std::recursive_mutex> lk(curl_mutex_);
|
||||
|
||||
@@ -64,6 +64,23 @@ public:
|
||||
const std::string& user, const std::string& password,
|
||||
bool useTls);
|
||||
|
||||
/**
|
||||
* @brief Set up the connection without probing it (connect() = open() + a getinfo probe)
|
||||
*
|
||||
* For sending "stop": the getinfo probe needs the daemon's cs_main, so against a busy node it
|
||||
* times out (holding one of the daemon's few RPC slots) or is refused, and stop never went out.
|
||||
* isConnected() stays false; call() works once this returns true.
|
||||
*/
|
||||
bool open(const std::string& host, const std::string& port,
|
||||
const std::string& user, const std::string& password, bool useTls);
|
||||
|
||||
/**
|
||||
* @brief Send "stop" once, with its own timeout; returns how it went (None = the daemon ran it)
|
||||
*
|
||||
* A ReplyTimeout here is NOT a failure: the request reached the daemon and waits for a worker.
|
||||
*/
|
||||
RpcFailureKind trySendStop(long timeoutSec = 2);
|
||||
|
||||
/**
|
||||
* @brief Disconnect from dragonxd
|
||||
*/
|
||||
@@ -265,6 +282,7 @@ private:
|
||||
// A non-200 reply: RpcError when the body carries a JSON-RPC error, else RpcTransportError classified
|
||||
// from the status and plain-text body. hideBody keeps the body out of the message (secret calls).
|
||||
[[noreturn]] static void throwHttpFailure(long httpCode, const std::string& body, bool hideBody);
|
||||
bool probeConnection(const std::string& host); // connect()'s getinfo probe, after open()
|
||||
void logConnectFailure(const std::string& msg);
|
||||
|
||||
// Splits a UnifiedCallback into the (Callback, ErrorCallback) pair used by doRPC.
|
||||
|
||||
@@ -587,6 +587,19 @@ void I18n::loadBuiltinEnglish()
|
||||
strings_["force_quit_confirm_title"] = "Force Quit?";
|
||||
strings_["force_quit_confirm_msg"] = "This will immediately kill the daemon without a clean shutdown.\nThis may corrupt the blockchain index and require a resync.";
|
||||
strings_["force_quit_yes"] = "Force Quit";
|
||||
// Stopping the node on quit: deliver stop, wait, never kill unasked (translations: S10 batch)
|
||||
strings_["shutdown_leave_running"] = "Close wallet — leave node running";
|
||||
strings_["shutdown_leave_running_tip"] = "Closes the wallet now. The node keeps running and finishes shutting down on its own; nothing is lost.";
|
||||
strings_["shutdown_kill_node"] = "Kill node (unsafe)";
|
||||
strings_["shutdown_kill_node_tip"] = "Ends the node at once. Anything it has not saved is lost, and its next start replays the blocks since its last save.";
|
||||
strings_["shutdown_kill_confirm_title"] = "Kill the node?";
|
||||
strings_["shutdown_kill_confirm_msg"] = "The node has shown no CPU or disk activity for a long time and may be stuck.\nKilling it loses everything it has not saved; its next start can take much longer.";
|
||||
strings_["shutdown_kill_confirm_yes"] = "Kill node";
|
||||
strings_["shutdown_stop_asking"] = "Asking the node to stop…";
|
||||
strings_["shutdown_stop_queued"] = "The node has the stop request but is busy — it will stop when it can";
|
||||
strings_["shutdown_stop_delivered"] = "The node is shutting down and saving its state";
|
||||
strings_["shutdown_stop_cpu_suffix"] = " · CPU %d%%";
|
||||
strings_["shutdown_stop_idle"] = "No CPU or disk activity for %d min — the node may be stuck";
|
||||
strings_["reduce_motion"] = "Reduce Motion";
|
||||
strings_["tt_reduce_motion"] = "Disable animated transitions and balance lerp for accessibility";
|
||||
strings_["ago"] = "ago";
|
||||
|
||||
@@ -80,6 +80,8 @@
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <new>
|
||||
#include <set>
|
||||
@@ -3431,6 +3433,165 @@ void testPosixDestructorLeavesProcessRunning()
|
||||
fsn::remove(script, ec);
|
||||
fsn::remove(pidFile, ec);
|
||||
}
|
||||
|
||||
// executeDaemonStop over a REAL EmbeddedDaemon child: steady clock, real waitpid/signals/CPU sampling.
|
||||
// No RPC here (stop attempts report NotListening), so on POSIX the group SIGTERM is the stop channel.
|
||||
class RealStopRuntime final : public dragonx::daemon::DaemonController::StopRuntime {
|
||||
public:
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
explicit RealStopRuntime(dragonx::daemon::EmbeddedDaemon& d) : d_(d), t0_(std::chrono::steady_clock::now()) {}
|
||||
double detachAt = -1.0; // choose "Leave running" at this time
|
||||
double maxCpuPercent = -1.0;
|
||||
double nowSec() override { return secondsSince(t0_); }
|
||||
bool processAlive() override { return d_.pollOwnedProcess(); }
|
||||
dragonx::daemon::StopSendResult trySendStop() override { return dragonx::daemon::StopSendResult::NotListening; }
|
||||
bool nextCredentials() override { return false; }
|
||||
bool sendGracefulSignal() override { return d_.signalGracefulStop(); }
|
||||
double cpuSeconds() override { return d_.ownedResourceUsage().cpuSeconds; }
|
||||
double ioBytes() override { return -1.0; }
|
||||
bool rebuildActive() override { return false; }
|
||||
DC::StopUserChoice userChoice() override
|
||||
{
|
||||
return (detachAt >= 0 && nowSec() >= detachAt) ? DC::StopUserChoice::Detach : DC::StopUserChoice::None;
|
||||
}
|
||||
void offerForceKill(bool) override {}
|
||||
void forceKill() override { d_.forceKillOwnedProcess(); }
|
||||
void setStatus(const DC::StopStatus& s) override { maxCpuPercent = std::max(maxCpuPercent, s.cpuPercent); }
|
||||
|
||||
private:
|
||||
dragonx::daemon::EmbeddedDaemon& d_;
|
||||
std::chrono::steady_clock::time_point t0_;
|
||||
};
|
||||
|
||||
class RealSleepTask final : public dragonx::daemon::DaemonController::LifecycleTaskContext {
|
||||
public:
|
||||
bool cancelled() const override { return false; }
|
||||
bool shuttingDown() const override { return false; }
|
||||
void sleepForMs(int ms) override { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); }
|
||||
};
|
||||
|
||||
pid_t readPidFile(const std::filesystem::path& pidFile)
|
||||
{
|
||||
for (int i = 0; i < 300; ++i) {
|
||||
std::ifstream in(pidFile);
|
||||
long v = 0;
|
||||
if (in >> v && v > 0) return static_cast<pid_t>(v);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// S1: "Leave running" on a node that ignores SIGTERM and keeps a core busy: the stop returns at the next
|
||||
// tick, nothing kills it, it is still alive afterwards, and its CPU was being measured.
|
||||
void testPosixStopSessionDetach()
|
||||
{
|
||||
using dragonx::daemon::EmbeddedDaemon;
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
namespace fsn = std::filesystem;
|
||||
const fsn::path pidFile = fsn::temp_directory_path() / "od_fake_daemon_detach.pid";
|
||||
std::error_code ec;
|
||||
fsn::remove(pidFile, ec);
|
||||
const auto script = writeFakeDaemonScript("od_fake_daemon_detach.sh",
|
||||
"trap '' TERM\necho $$ > \"$OD_TEST_PIDFILE\"\nwhile :; do :; done\n"); // pid file = trap is set
|
||||
setenv("OD_TEST_PIDFILE", pidFile.string().c_str(), 1);
|
||||
|
||||
EmbeddedDaemon d;
|
||||
d.setSkipPortCheck(true);
|
||||
EXPECT_TRUE(d.start(script.string()));
|
||||
const pid_t pid = readPidFile(pidFile);
|
||||
EXPECT_TRUE(pid > 0);
|
||||
EXPECT_TRUE(d.beginStop());
|
||||
RealStopRuntime rt(d);
|
||||
rt.detachAt = 2.0;
|
||||
RealSleepTask task;
|
||||
DC::StopPolicy p;
|
||||
p.gracefulSignal = true;
|
||||
p.userChoices = true;
|
||||
const auto outcome = DC::executeDaemonStop(p, rt, task);
|
||||
EXPECT_TRUE(outcome == DC::StopOutcome::Detached);
|
||||
EXPECT_TRUE(rt.nowSec() < 2.6);
|
||||
EXPECT_TRUE(rt.maxCpuPercent > 20.0); // the busy loop was measured
|
||||
d.endStop(/*keepWatching=*/false);
|
||||
EXPECT_TRUE(pid > 0 && kill(pid, 0) == 0); // left running
|
||||
EXPECT_FALSE(d.isRunning());
|
||||
|
||||
if (pid > 0) {
|
||||
kill(-pid, SIGKILL);
|
||||
// endStop() left a detached reaper waiting on it: the zombie disappears once it is reaped.
|
||||
bool reaped = false;
|
||||
for (int i = 0; i < 200 && !reaped; ++i) {
|
||||
reaped = (kill(pid, 0) != 0);
|
||||
if (!reaped) std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
EXPECT_TRUE(reaped);
|
||||
}
|
||||
unsetenv("OD_TEST_PIDFILE");
|
||||
fsn::remove(script, ec);
|
||||
fsn::remove(pidFile, ec);
|
||||
}
|
||||
|
||||
// S1/S2 on POSIX: with no RPC answer, the group SIGTERM stops a node that handles it, promptly and
|
||||
// without counting a crash.
|
||||
void testPosixStopSessionSigterm()
|
||||
{
|
||||
using dragonx::daemon::EmbeddedDaemon;
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
const auto script = writeFakeDaemonScript("od_fake_daemon_session_term.sh",
|
||||
"trap 'exit 0' TERM\nwhile :; do sleep 0.1; done\n");
|
||||
EmbeddedDaemon d;
|
||||
d.setSkipPortCheck(true);
|
||||
EXPECT_TRUE(d.start(script.string()));
|
||||
EXPECT_TRUE(d.beginStop());
|
||||
RealStopRuntime rt(d);
|
||||
RealSleepTask task;
|
||||
DC::StopPolicy p;
|
||||
p.gracefulSignal = true;
|
||||
p.userChoices = true;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == DC::StopOutcome::Exited);
|
||||
EXPECT_TRUE(rt.nowSec() < 2.0);
|
||||
d.endStop(/*keepWatching=*/false);
|
||||
EXPECT_TRUE(d.getState() == EmbeddedDaemon::State::Stopped);
|
||||
EXPECT_EQ(d.getCrashCount(), 0);
|
||||
std::error_code ec; std::filesystem::remove(script, ec);
|
||||
}
|
||||
|
||||
// S1: a bounded wait that gives up keeps watching the node: its later exit is still seen, and booked
|
||||
// as the clean stop we asked for (not a crash).
|
||||
void testPosixStopSessionGiveUpKeepsWatching()
|
||||
{
|
||||
using dragonx::daemon::EmbeddedDaemon;
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
namespace fsn = std::filesystem;
|
||||
const fsn::path readyFile = fsn::temp_directory_path() / "od_fake_daemon_session_slow.ready";
|
||||
std::error_code rmEc;
|
||||
fsn::remove(readyFile, rmEc);
|
||||
// Signal readiness only once the trap is installed: a SIGTERM before that line would kill it.
|
||||
const auto script = writeFakeDaemonScript("od_fake_daemon_session_slow.sh",
|
||||
"trap '' TERM\necho $$ > \"$OD_TEST_PIDFILE\"\nsleep 1.5\nexit 0\n");
|
||||
setenv("OD_TEST_PIDFILE", readyFile.string().c_str(), 1);
|
||||
EmbeddedDaemon d;
|
||||
d.setSkipPortCheck(true);
|
||||
EXPECT_TRUE(d.start(script.string()));
|
||||
EXPECT_TRUE(readPidFile(readyFile) > 0);
|
||||
EXPECT_TRUE(d.beginStop());
|
||||
RealStopRuntime rt(d);
|
||||
RealSleepTask task;
|
||||
DC::StopPolicy p;
|
||||
p.gracefulSignal = true;
|
||||
p.maxWaitSec = 0.5;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == DC::StopOutcome::StillRunning);
|
||||
d.endStop(/*keepWatching=*/true);
|
||||
EXPECT_TRUE(d.isRunning());
|
||||
for (int i = 0; i < 400 && d.getState() != EmbeddedDaemon::State::Stopped; ++i)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
EXPECT_TRUE(d.getState() == EmbeddedDaemon::State::Stopped);
|
||||
EXPECT_EQ(d.getCrashCount(), 0);
|
||||
d.stop(1000); // join the resumed monitor
|
||||
unsetenv("OD_TEST_PIDFILE");
|
||||
std::error_code ec;
|
||||
fsn::remove(script, ec);
|
||||
fsn::remove(readyFile, ec);
|
||||
}
|
||||
#endif // !_WIN32
|
||||
|
||||
#ifdef _WIN32
|
||||
@@ -3481,6 +3642,380 @@ void testWindowsStopNeverKillsForeignDaemon()
|
||||
}
|
||||
#endif // _WIN32
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
// S2 + S1: delivering "stop" and the never-kill wait (DaemonController::executeDaemonStop), driven by a
|
||||
// scripted runtime on a virtual clock.
|
||||
// ---------------------------------------------------------------------------------------------------
|
||||
|
||||
void testStopDeliveryPolicy()
|
||||
{
|
||||
using dragonx::daemon::StopDeliveryPolicy;
|
||||
using dragonx::daemon::StopSendResult;
|
||||
using R = StopSendResult;
|
||||
{ // Refused before queueing: not delivered, cheap to retry after 1.5s.
|
||||
StopDeliveryPolicy p(true);
|
||||
EXPECT_TRUE(p.shouldSend(0.0));
|
||||
EXPECT_TRUE(p.record(R::QueueFull, 0.0) == R::QueueFull);
|
||||
EXPECT_FALSE(p.shouldSend(1.0));
|
||||
EXPECT_TRUE(p.shouldSend(1.5));
|
||||
EXPECT_FALSE(p.delivered());
|
||||
}
|
||||
{ // Queued: the daemon has it; no resend for 30s (each resend takes a queue slot).
|
||||
StopDeliveryPolicy p(true);
|
||||
p.record(R::Queued, 0.0);
|
||||
EXPECT_TRUE(p.heard());
|
||||
EXPECT_FALSE(p.delivered());
|
||||
EXPECT_FALSE(p.shouldSend(29.9));
|
||||
EXPECT_TRUE(p.shouldSend(30.0));
|
||||
// ...and a refusal after that means it unbound its listener to shut down.
|
||||
EXPECT_TRUE(p.record(R::NotListening, 30.0) == R::ShuttingDown);
|
||||
EXPECT_TRUE(p.delivered());
|
||||
}
|
||||
{ // NotListening before we were ever heard is just "not up / not bound": retry.
|
||||
StopDeliveryPolicy p(true);
|
||||
EXPECT_TRUE(p.record(R::NotListening, 0.0) == R::NotListening);
|
||||
EXPECT_FALSE(p.delivered());
|
||||
EXPECT_TRUE(p.shouldSend(1.5));
|
||||
}
|
||||
{ // Accepted / ShuttingDown: delivered, never resend.
|
||||
StopDeliveryPolicy a(true), s(true);
|
||||
a.record(R::Accepted, 0.0);
|
||||
s.record(R::ShuttingDown, 0.0);
|
||||
EXPECT_TRUE(a.delivered() && a.finished() && !a.shouldSend(1000.0));
|
||||
EXPECT_TRUE(s.delivered() && s.finished() && !s.shouldSend(1000.0));
|
||||
}
|
||||
{ // Auth: terminal for a node we don't own; our own may retry with other credentials.
|
||||
StopDeliveryPolicy unowned(false), owned(true);
|
||||
unowned.record(R::Auth, 0.0);
|
||||
EXPECT_TRUE(unowned.finished() && !unowned.delivered());
|
||||
owned.record(R::Auth, 0.0);
|
||||
EXPECT_FALSE(owned.finished());
|
||||
owned.retryNow(0.2);
|
||||
EXPECT_TRUE(owned.shouldSend(0.2));
|
||||
}
|
||||
using dragonx::daemon::stopSendResultFromKind;
|
||||
using K = dragonx::rpc::RpcFailureKind;
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::None) == R::Accepted);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::ReplyTimeout) == R::Queued);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::QueueFull) == R::QueueFull);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::NotListening) == R::NotListening);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::ConnectTimeout) == R::NotListening);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::ShuttingDown) == R::ShuttingDown);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::Unhandled) == R::ShuttingDown);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::Auth) == R::Auth);
|
||||
EXPECT_TRUE(stopSendResultFromKind(K::Aborted) == R::Other);
|
||||
}
|
||||
|
||||
void testProcStatParse()
|
||||
{
|
||||
using dragonx::daemon::EmbeddedDaemon;
|
||||
long pgrp = 0;
|
||||
unsigned long long ticks = 0;
|
||||
// comm with spaces AND a ')': fields must be read after the LAST ')'. tpgid -1 must parse.
|
||||
EXPECT_TRUE(EmbeddedDaemon::parseProcStatLine(
|
||||
"4242 (weird) name) S 1 4240 4240 0 -1 4194560 100 0 0 0 250 50 0 0 20 0 1 0 12345 1000 10",
|
||||
pgrp, ticks));
|
||||
EXPECT_EQ(pgrp, 4240L);
|
||||
EXPECT_EQ(ticks, 300ULL);
|
||||
EXPECT_FALSE(EmbeddedDaemon::parseProcStatLine("garbage without parens", pgrp, ticks));
|
||||
EXPECT_FALSE(EmbeddedDaemon::parseProcStatLine("1 (x) S 1", pgrp, ticks));
|
||||
}
|
||||
|
||||
void testStopProgressTracker()
|
||||
{
|
||||
using Tracker = dragonx::daemon::DaemonController::StopProgressTracker;
|
||||
{ // Flat CPU and I/O: idle time grows from the first sample.
|
||||
Tracker tr;
|
||||
for (int s = 0; s <= 300; ++s) tr.add(s, 10.0, 5e6);
|
||||
EXPECT_NEAR(tr.idleSec(300.0), 300.0, 1e-9);
|
||||
}
|
||||
{ // 0.1 cores is progress (threshold 0.05).
|
||||
Tracker tr;
|
||||
for (int s = 0; s <= 300; ++s) tr.add(s, 0.1 * s, -1.0);
|
||||
EXPECT_TRUE(tr.idleSec(300.0) < 1.0);
|
||||
EXPECT_NEAR(tr.cpuPercent(), 10.0, 1e-6);
|
||||
}
|
||||
{ // A low-CPU flush still writing ~2 MB/min is progress too.
|
||||
Tracker tr;
|
||||
for (int s = 0; s <= 300; ++s) tr.add(s, 1.0, 2e6 / 60.0 * s);
|
||||
EXPECT_TRUE(tr.idleSec(300.0) < 1.0);
|
||||
}
|
||||
{ // No readings at all: never "idle", so Kill is never offered.
|
||||
Tracker tr;
|
||||
for (int s = 0; s <= 3600; s += 10) tr.add(s, -1.0, -1.0);
|
||||
EXPECT_NEAR(tr.idleSec(3600.0), 0.0, 1e-9);
|
||||
EXPECT_TRUE(tr.cpuPercent() < 0);
|
||||
}
|
||||
}
|
||||
|
||||
void testEvaluateStopWait()
|
||||
{
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
using D = DC::StopWaitDecision;
|
||||
DC::StopPolicy shutdown; // the shutdown screen: owned, unbounded, choices
|
||||
shutdown.userChoices = true;
|
||||
DC::StopPolicy background; // a restart/rescan/switch: bounded, no choices
|
||||
background.maxWaitSec = 120;
|
||||
background.abandonOnShutdown = true;
|
||||
auto sample = [] { DC::StopWaitSample s; return s; };
|
||||
|
||||
auto s = sample(); s.processAlive = false;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Done);
|
||||
s = sample(); s.choice = DC::StopUserChoice::Detach;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Detach);
|
||||
EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::Wait); // no choices there
|
||||
s = sample(); s.choice = DC::StopUserChoice::Kill; // Kill without an offer: ignored
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait);
|
||||
s.killOffered = true;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::ForceKill);
|
||||
s = sample(); s.delivered = true; s.idleSec = 600; // delivered + idle 10 min: offer
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::OfferForceKill);
|
||||
s.rebuildActive = true; // ...never during a rebuild
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait);
|
||||
s = sample(); s.delivered = false; s.idleSec = 6000; // never delivered: no offer
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait);
|
||||
s = sample(); s.elapsedSec = 120;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::GiveUp);
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); // the shutdown screen has no budget
|
||||
s = sample(); s.shuttingDown = true;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::GiveUp);
|
||||
EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait);
|
||||
DC::StopPolicy unowned = shutdown;
|
||||
unowned.owned = false;
|
||||
s = sample(); s.delivered = true; s.idleSec = 6000; s.killOffered = true; s.choice = DC::StopUserChoice::Kill;
|
||||
EXPECT_TRUE(DC::evaluateStopWait(unowned, s) == D::Wait); // never kill what we didn't launch
|
||||
}
|
||||
|
||||
// A scripted node for executeDaemonStop, on a virtual clock (seconds) that sleeps and sends advance.
|
||||
class MockStopRuntime final : public dragonx::daemon::DaemonController::StopRuntime {
|
||||
public:
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
using R = dragonx::daemon::StopSendResult;
|
||||
|
||||
double clock = 0.0;
|
||||
std::vector<R> sends; // results in order; the last one repeats
|
||||
double exitAt = -1.0; // the process exits at this time (< 0: not on its own)
|
||||
double exitAfterDelivery = -1.0; // ...or this long after stop/SIGTERM reached it
|
||||
bool hasSignal = false; // POSIX-like graceful channel
|
||||
int credentialsLeft = 0;
|
||||
std::function<double(double)> cpuAt; // cumulative CPU seconds at t; empty = unknown
|
||||
std::function<DC::StopUserChoice(double)> choiceAt;
|
||||
bool rebuild = false;
|
||||
|
||||
std::vector<std::string> calls; // "stop", "sigterm", "creds", "offer", "withdraw", "kill"
|
||||
double deliveredAt = -1.0, firstSigtermAt = -1.0, firstOfferAt = -1.0;
|
||||
double lastStopAt = -1.0;
|
||||
bool killed = false;
|
||||
bool hitSafetyLimit = false;
|
||||
DC::StopStatus lastStatus;
|
||||
|
||||
int count(const std::string& c) const { return (int)std::count(calls.begin(), calls.end(), c); }
|
||||
|
||||
double nowSec() override { return clock; }
|
||||
bool processAlive() override
|
||||
{
|
||||
if (clock > 36000.0) { hitSafetyLimit = true; return false; } // a test bug, not an outcome
|
||||
if (killed) return false;
|
||||
if (exitAt >= 0 && clock >= exitAt) return false;
|
||||
if (exitAfterDelivery >= 0 && deliveredAt >= 0 && clock >= deliveredAt + exitAfterDelivery) return false;
|
||||
return true;
|
||||
}
|
||||
R trySendStop() override
|
||||
{
|
||||
calls.push_back("stop");
|
||||
lastStopAt = clock;
|
||||
const size_t n = calls.empty() ? 0 : (size_t)count("stop") - 1;
|
||||
const R r = sends.empty() ? R::Accepted : sends[std::min(n, sends.size() - 1)];
|
||||
clock += (r == R::Queued) ? 2.0 : 0.01; // a reply timeout costs the client timeout
|
||||
if ((r == R::Accepted || r == R::ShuttingDown) && deliveredAt < 0) deliveredAt = clock;
|
||||
return r;
|
||||
}
|
||||
bool nextCredentials() override
|
||||
{
|
||||
calls.push_back("creds");
|
||||
return credentialsLeft-- > 0;
|
||||
}
|
||||
bool sendGracefulSignal() override
|
||||
{
|
||||
if (!hasSignal) return false;
|
||||
calls.push_back("sigterm");
|
||||
if (firstSigtermAt < 0) firstSigtermAt = clock;
|
||||
if (deliveredAt < 0) deliveredAt = clock;
|
||||
return true;
|
||||
}
|
||||
double cpuSeconds() override { return cpuAt ? cpuAt(clock) : -1.0; }
|
||||
double ioBytes() override { return -1.0; }
|
||||
bool rebuildActive() override { return rebuild; }
|
||||
DC::StopUserChoice userChoice() override { return choiceAt ? choiceAt(clock) : DC::StopUserChoice::None; }
|
||||
void offerForceKill(bool offered) override
|
||||
{
|
||||
calls.push_back(offered ? "offer" : "withdraw");
|
||||
if (offered && firstOfferAt < 0) firstOfferAt = clock;
|
||||
}
|
||||
void forceKill() override
|
||||
{
|
||||
calls.push_back("kill");
|
||||
killed = true;
|
||||
}
|
||||
void setStatus(const DC::StopStatus& s) override { lastStatus = s; }
|
||||
};
|
||||
|
||||
class MockStopTask final : public dragonx::daemon::DaemonController::LifecycleTaskContext {
|
||||
public:
|
||||
explicit MockStopTask(MockStopRuntime& rt) : rt_(rt) {}
|
||||
double shutdownAt = -1.0;
|
||||
bool cancelled() const override { return false; }
|
||||
bool shuttingDown() const override { return shutdownAt >= 0 && rt_.clock >= shutdownAt; }
|
||||
void sleepForMs(int ms) override { rt_.clock += ms / 1000.0; }
|
||||
|
||||
private:
|
||||
MockStopRuntime& rt_;
|
||||
};
|
||||
|
||||
dragonx::daemon::DaemonController::StopPolicy shutdownStopPolicy(bool posix)
|
||||
{
|
||||
dragonx::daemon::DaemonController::StopPolicy p;
|
||||
p.owned = true;
|
||||
p.gracefulSignal = posix;
|
||||
p.userChoices = true;
|
||||
return p;
|
||||
}
|
||||
|
||||
void testDaemonStopExecution()
|
||||
{
|
||||
using DC = dragonx::daemon::DaemonController;
|
||||
using R = dragonx::daemon::StopSendResult;
|
||||
using O = DC::StopOutcome;
|
||||
|
||||
{ // Already exited: not a single RPC call.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.exitAt = 0.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_TRUE(rt.calls.empty());
|
||||
}
|
||||
{ // The 2026-09-30 12:21:44 -> 12:22:04 regression: the queue is full for a while. Retry until it
|
||||
// lands (Windows-like, no signal channel), then wait for the exit. Never kill.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::QueueFull, R::QueueFull, R::QueueFull, R::QueueFull, R::QueueFull, R::Accepted};
|
||||
rt.exitAfterDelivery = 3.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("stop"), 6);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
EXPECT_TRUE(rt.deliveredAt >= 5 * 1.5); // retries were spaced, not a tight loop
|
||||
EXPECT_FALSE(rt.hitSafetyLimit);
|
||||
}
|
||||
{ // Never automatic: queue full for an hour, CPU busy (a rebuild with no log marker). No kill, no
|
||||
// offer; the node exits on its own at 60 min.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::QueueFull};
|
||||
rt.exitAt = 3600.0;
|
||||
rt.cpuAt = [](double t) { return 6.0 * t; };
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
EXPECT_EQ(rt.count("offer"), 0);
|
||||
EXPECT_TRUE(rt.count("stop") > 1000); // still trying every 1.5s
|
||||
EXPECT_FALSE(rt.hitSafetyLimit);
|
||||
}
|
||||
{ // POSIX: the first attempt is refused, so the group SIGTERM goes out at once (<= 2s), and no
|
||||
// more RPC attempts are needed.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::QueueFull};
|
||||
rt.hasSignal = true;
|
||||
rt.exitAfterDelivery = 2.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited);
|
||||
EXPECT_TRUE(rt.firstSigtermAt >= 0 && rt.firstSigtermAt <= 2.0);
|
||||
EXPECT_EQ(rt.count("stop"), 1);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
}
|
||||
{ // Queued (reply timeout) is not a failure: no SIGTERM, no resend before 30s.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::Queued};
|
||||
rt.hasSignal = true;
|
||||
rt.exitAt = 25.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("stop"), 1);
|
||||
EXPECT_EQ(rt.count("sigterm"), 0);
|
||||
}
|
||||
{ // Delivered, then flat CPU: Kill is OFFERED after ~10 min, and only an explicit Kill kills (once).
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.cpuAt = [](double) { return 42.0; };
|
||||
rt.choiceAt = [](double t) { return t >= 900.0 ? DC::StopUserChoice::Kill : DC::StopUserChoice::None; };
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Killed);
|
||||
EXPECT_TRUE(rt.firstOfferAt >= 600.0 && rt.firstOfferAt < 602.0);
|
||||
EXPECT_EQ(rt.count("kill"), 1);
|
||||
EXPECT_TRUE(rt.clock >= 900.0); // waited for the user, didn't act on the offer itself
|
||||
}
|
||||
{ // ...with no answer to the offer, it keeps waiting (and never kills).
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.cpuAt = [](double) { return 42.0; };
|
||||
rt.exitAt = 3600.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("offer"), 1);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
}
|
||||
{ // A rebuild in progress suppresses the offer even with flat readings.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.cpuAt = [](double) { return 42.0; };
|
||||
rt.rebuild = true;
|
||||
rt.exitAt = 1800.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("offer"), 0);
|
||||
}
|
||||
{ // "Leave running": returns at once, no kill, no further stop attempts.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::QueueFull};
|
||||
rt.choiceAt = [](double t) { return t >= 5.0 ? DC::StopUserChoice::Detach : DC::StopUserChoice::None; };
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Detached);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
EXPECT_TRUE(rt.lastStopAt < 5.0);
|
||||
EXPECT_TRUE(rt.clock < 5.5);
|
||||
}
|
||||
{ // A node we didn't launch is never killed, even with Kill chosen; a bounded wait then gives up.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
DC::StopPolicy p = shutdownStopPolicy(false);
|
||||
p.owned = false;
|
||||
p.maxWaitSec = 120.0;
|
||||
rt.choiceAt = [](double) { return DC::StopUserChoice::Kill; };
|
||||
EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == O::StillRunning);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
EXPECT_EQ(rt.count("offer"), 0);
|
||||
EXPECT_TRUE(rt.clock >= 120.0 && rt.clock < 121.0);
|
||||
}
|
||||
{ // Background callers (restart, rescan, switch): bounded and never kill; they also let go as soon
|
||||
// as the app starts shutting down, so the shutdown can take over.
|
||||
DC::StopPolicy bg;
|
||||
bg.maxWaitSec = 120.0;
|
||||
bg.abandonOnShutdown = true;
|
||||
MockStopRuntime a; MockStopTask ta(a);
|
||||
a.cpuAt = [](double) { return 1.0; };
|
||||
EXPECT_TRUE(DC::executeDaemonStop(bg, a, ta) == O::StillRunning);
|
||||
EXPECT_EQ(a.count("kill"), 0);
|
||||
EXPECT_TRUE(a.clock >= 120.0 && a.clock < 121.0);
|
||||
MockStopRuntime b; MockStopTask tb(b);
|
||||
tb.shutdownAt = 10.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(bg, b, tb) == O::StillRunning);
|
||||
EXPECT_TRUE(b.clock >= 10.0 && b.clock < 11.0);
|
||||
}
|
||||
{ // Our own node refusing our credentials: try the next source, then deliver.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::Auth, R::Auth, R::Accepted};
|
||||
rt.credentialsLeft = 2;
|
||||
rt.exitAfterDelivery = 1.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("creds"), 2);
|
||||
EXPECT_EQ(rt.count("stop"), 3);
|
||||
}
|
||||
{ // ...out of credential sources on POSIX: the graceful signal still stops it.
|
||||
MockStopRuntime rt; MockStopTask task(rt);
|
||||
rt.sends = {R::Auth};
|
||||
rt.hasSignal = true;
|
||||
rt.exitAfterDelivery = 1.0;
|
||||
EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited);
|
||||
EXPECT_EQ(rt.count("sigterm"), 1);
|
||||
EXPECT_EQ(rt.count("kill"), 0);
|
||||
}
|
||||
}
|
||||
|
||||
// S3: on Linux a node in its final shutdown phase reads comm "hush-shutoff"; its executable still says
|
||||
// dragonxd, so it must still count as running (it holds the datadir lock until it exits).
|
||||
void testDaemonProcessIdentity()
|
||||
@@ -8064,6 +8599,11 @@ int main()
|
||||
testDaemonShutdownPolicy();
|
||||
testDatadirLockGate();
|
||||
testDaemonProcessIdentity();
|
||||
testStopDeliveryPolicy();
|
||||
testProcStatParse();
|
||||
testStopProgressTracker();
|
||||
testEvaluateStopWait();
|
||||
testDaemonStopExecution();
|
||||
#ifdef _WIN32
|
||||
testWindowsStopNeverKillsForeignDaemon();
|
||||
#endif
|
||||
@@ -8074,6 +8614,9 @@ int main()
|
||||
testPosixRequestedStopIsNotACrash();
|
||||
testPosixStopSigtermIsGraceful();
|
||||
testPosixDestructorLeavesProcessRunning();
|
||||
testPosixStopSessionDetach();
|
||||
testPosixStopSessionSigterm();
|
||||
testPosixStopSessionGiveUpKeepsWatching();
|
||||
#endif
|
||||
testPlatformEnsureDirectory();
|
||||
testVerifySaplingParams();
|
||||
|
||||
Reference in New Issue
Block a user