fix(daemon): let POSIX stop() see the daemon exit instead of always SIGKILLing
POSIX stop() set should_stop_, which made the monitor thread return without reaping the child, and then polled the state-based isRunning(), which stays true while the state is Stopping. So every Linux/macOS stop waited out its whole grace period, sent SIGTERM, waited 10 s more and SIGKILLed, even when the node had exited seconds earlier. Closing the GUI with an idle node took ~30 s; a node still flushing at 30 s was killed. - stop() joins the monitor first, then polls waitpid(pid, WNOHANG) on a cached pid, so it is the only reaper and sees the exit at once. - markStopRequested() (set by App::stopEmbeddedDaemon before the RPC stop, and by stop() itself; cleared by start()) makes an exit that lands before stop() runs a clean stop instead of a crash, on POSIX and in the Windows monitor. crash_count_ drives auto-restart, and a crash recorded before the request keeps its count. - The SIGTERM -> SIGKILL escalation stays for now, but only fires for a daemon that has really not exited. Replacing it with a wait policy and an explicit user choice is follow-up work. Verified against the fake daemon: an idle node that exits 3.4 s after stop used to be SIGTERMed at +20 s and SIGKILLed at +30 s (GUI closed in 30.6 s); it is now reaped as "exited cleanly" ~50 ms after it exits (GUI closes in 4.0 s). New POSIX tests cover the clean exit, the early requested exit and a graceful SIGTERM, each time-bounded so the old code fails fast. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -5395,6 +5395,10 @@ void App::stopEmbeddedDaemon()
|
||||
// for it to disappear (handled after the stop() call).
|
||||
const bool owned = daemon_controller_->isRunning();
|
||||
|
||||
// Any exit from here on is the stop we are asking for, not a crash — even one that lands before
|
||||
// daemon_controller_->stop() below gets to reap it (crash_count_ drives auto-restart / give-up).
|
||||
daemon_controller_->markStopRequested();
|
||||
|
||||
// Send RPC "stop" command — this is the graceful path that lets the
|
||||
// daemon flush state, save block indexes, close sockets, etc.
|
||||
bool stop_sent = false;
|
||||
|
||||
@@ -52,6 +52,11 @@ void DaemonController::stop(int waitMs)
|
||||
daemon_->stop(waitMs);
|
||||
}
|
||||
|
||||
void DaemonController::markStopRequested()
|
||||
{
|
||||
daemon_->markStopRequested();
|
||||
}
|
||||
|
||||
bool DaemonController::isRunning() const
|
||||
{
|
||||
return daemon_->isRunning();
|
||||
|
||||
@@ -90,6 +90,7 @@ public:
|
||||
|
||||
bool start(const config::Settings* settings);
|
||||
void stop(int waitMs);
|
||||
void markStopRequested();
|
||||
|
||||
bool isRunning() const;
|
||||
bool externalDaemonDetected() const;
|
||||
|
||||
@@ -602,6 +602,7 @@ bool EmbeddedDaemon::start(const std::string& binary_path)
|
||||
override_datadir_.clear();
|
||||
override_extra_args_.clear();
|
||||
|
||||
stop_requested_ = false; // a new process: its exits are crashes until a stop is asked for
|
||||
if (!startProcess(daemon_path, args)) {
|
||||
// startProcess() sets a precise last_error_ (e.g. "dragonxd could not be executed:
|
||||
// ... not executable or wrong architecture"). Surface THAT via setState — which also
|
||||
@@ -971,6 +972,13 @@ void EmbeddedDaemon::monitorProcess()
|
||||
// Read any remaining output after process exits
|
||||
drainOutput();
|
||||
|
||||
// An exit after markStopRequested() is the stop we asked for (it can land before stop() runs).
|
||||
if (!should_stop_ && !isRunning() && stop_requested_) {
|
||||
DEBUG_LOGF("dragonxd exited after a requested stop\n");
|
||||
setState(State::Stopped, "dragonxd stopped");
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if process exited unexpectedly
|
||||
if (!should_stop_ && !isRunning()) {
|
||||
// Get exit code for diagnostics
|
||||
@@ -1308,79 +1316,80 @@ void EmbeddedDaemon::drainOutput()
|
||||
}
|
||||
}
|
||||
|
||||
static std::string describeExitStatus(int status)
|
||||
{
|
||||
if (WIFEXITED(status)) return "exit code: " + std::to_string(WEXITSTATUS(status));
|
||||
if (WIFSIGNALED(status)) return "killed by signal " + std::to_string(WTERMSIG(status));
|
||||
return "unknown exit status";
|
||||
}
|
||||
|
||||
bool EmbeddedDaemon::reapWithin(pid_t pid, int timeout_ms, int& status)
|
||||
{
|
||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms);
|
||||
for (;;) {
|
||||
drainOutput();
|
||||
const pid_t r = waitpid(pid, &status, WNOHANG);
|
||||
if (r == pid) return true;
|
||||
if (r < 0 && errno != EINTR) return true; // ECHILD: nothing left to reap
|
||||
if (std::chrono::steady_clock::now() >= deadline) return false;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::stop(int wait_ms)
|
||||
{
|
||||
stop_requested_ = true;
|
||||
should_stop_ = true;
|
||||
|
||||
if (process_pid_ > 0) {
|
||||
|
||||
// Join the monitor FIRST (it sees should_stop_ within one 100ms tick) so that from here on this is
|
||||
// the only reaper. isRunning() is state-based and stays true until we set Stopped below, so it can't
|
||||
// tell us the child exited: poll waitpid() on a cached pid instead. If the child exited inside that
|
||||
// tick, the monitor already reaped it and, seeing stop_requested_, recorded a clean stop (pid now 0).
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
|
||||
const pid_t pid = process_pid_;
|
||||
if (pid > 0) {
|
||||
setState(State::Stopping, "Stopping dragonxd...");
|
||||
|
||||
int status = 0;
|
||||
|
||||
// Phase 1: Wait for the daemon to exit naturally.
|
||||
// The caller (stopEmbeddedDaemon) already sent an RPC "stop" which
|
||||
// tells the daemon to flush LevelDB, close sockets, and exit cleanly.
|
||||
// On macOS/APFS the LevelDB flush can take several seconds — we must
|
||||
// NOT send SIGTERM until the daemon has had enough time to finish.
|
||||
DEBUG_LOGF("Waiting up to %d ms for daemon to exit after RPC stop...\n", wait_ms);
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
while (isRunning()) {
|
||||
drainOutput();
|
||||
|
||||
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
|
||||
if (elapsed >= wait_ms) {
|
||||
break;
|
||||
}
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
|
||||
// Phase 2: If still running, send SIGTERM and wait a further 10s.
|
||||
if (isRunning()) {
|
||||
DEBUG_LOGF("Daemon did not exit gracefully — sending SIGTERM to process group -%d\n", process_pid_);
|
||||
kill(-process_pid_, SIGTERM);
|
||||
|
||||
auto sigterm_start = std::chrono::steady_clock::now();
|
||||
while (isRunning()) {
|
||||
drainOutput();
|
||||
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - sigterm_start).count();
|
||||
if (elapsed >= 10000) {
|
||||
// Phase 3: Force kill
|
||||
DEBUG_LOGF("Forcing dragonxd termination with SIGKILL (group -%d)...\n", process_pid_);
|
||||
kill(-process_pid_, SIGKILL);
|
||||
break;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
if (reapWithin(pid, wait_ms, status)) {
|
||||
DEBUG_LOGF("Daemon exited cleanly after RPC stop (%s)\n", describeExitStatus(status).c_str());
|
||||
} else {
|
||||
DEBUG_LOGF("Daemon exited cleanly after RPC stop\n");
|
||||
// Phase 2: still running — SIGTERM (dragonxd treats it like RPC stop) and a further 10s.
|
||||
DEBUG_LOGF("Daemon did not exit gracefully — sending SIGTERM to process group -%d\n", pid);
|
||||
kill(-pid, SIGTERM);
|
||||
if (!reapWithin(pid, 10000, status)) {
|
||||
// Phase 3: Force kill
|
||||
DEBUG_LOGF("Forcing dragonxd termination with SIGKILL (group -%d)...\n", pid);
|
||||
kill(-pid, SIGKILL);
|
||||
while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {}
|
||||
}
|
||||
DEBUG_LOGF("dragonxd exited (%s)\n", describeExitStatus(status).c_str());
|
||||
}
|
||||
|
||||
drainOutput(); // read any remaining output
|
||||
|
||||
// Reap the child process
|
||||
int status;
|
||||
waitpid(process_pid_, &status, 0);
|
||||
process_pid_ = 0;
|
||||
}
|
||||
|
||||
|
||||
if (stdout_fd_ >= 0) {
|
||||
close(stdout_fd_);
|
||||
stdout_fd_ = -1;
|
||||
}
|
||||
|
||||
if (monitor_thread_.joinable()) {
|
||||
monitor_thread_.join();
|
||||
}
|
||||
|
||||
|
||||
setState(State::Stopped, "dragonxd stopped");
|
||||
}
|
||||
|
||||
void EmbeddedDaemon::monitorProcess()
|
||||
{
|
||||
char buffer[4096];
|
||||
|
||||
|
||||
while (!should_stop_) {
|
||||
// Check if process exited (non-blocking waitpid captures exit status)
|
||||
if (process_pid_ > 0) {
|
||||
@@ -1390,16 +1399,16 @@ void EmbeddedDaemon::monitorProcess()
|
||||
// Process exited — drain remaining output
|
||||
drainOutput();
|
||||
|
||||
crash_count_++;
|
||||
std::string exitInfo;
|
||||
if (WIFEXITED(status)) {
|
||||
exitInfo = "exit code: " + std::to_string(WEXITSTATUS(status));
|
||||
} else if (WIFSIGNALED(status)) {
|
||||
exitInfo = "killed by signal " + std::to_string(WTERMSIG(status));
|
||||
} else {
|
||||
exitInfo = "unknown exit status";
|
||||
}
|
||||
const std::string exitInfo = describeExitStatus(status);
|
||||
process_pid_ = 0;
|
||||
if (stop_requested_) {
|
||||
// The stop we asked for (markStopRequested() / stop()) landed before stop() could
|
||||
// reap it: a clean stop, not a crash. A crash recorded earlier keeps its count.
|
||||
DEBUG_LOGF("dragonxd exited after a requested stop (%s)\n", exitInfo.c_str());
|
||||
setState(State::Stopped, "dragonxd stopped");
|
||||
return;
|
||||
}
|
||||
crash_count_++;
|
||||
setState(State::Error, "dragonxd exited unexpectedly (" + exitInfo + ")");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -59,7 +59,16 @@ public:
|
||||
* @param wait_ms Maximum milliseconds to wait for graceful shutdown
|
||||
*/
|
||||
void stop(int wait_ms = 5000);
|
||||
|
||||
|
||||
/**
|
||||
* @brief Record that we are about to ask the daemon to stop (e.g. over RPC)
|
||||
*
|
||||
* Call it before sending the stop. If the daemon then exits before stop() runs, the monitor
|
||||
* records a clean stop instead of a crash (crash_count_ drives auto-restart). stop() sets it
|
||||
* too; start() clears it.
|
||||
*/
|
||||
void markStopRequested() { stop_requested_ = true; }
|
||||
|
||||
/**
|
||||
* @brief Check if daemon process is running
|
||||
*/
|
||||
@@ -306,10 +315,15 @@ private:
|
||||
#else
|
||||
pid_t process_pid_ = 0;
|
||||
int stdout_fd_ = -1;
|
||||
|
||||
// Waits up to timeout_ms for `pid` to exit, draining its output; true once reaped. Only valid
|
||||
// after monitor_thread_ is joined, so that stop() is the sole reaper.
|
||||
bool reapWithin(pid_t pid, int timeout_ms, int& status);
|
||||
#endif
|
||||
|
||||
|
||||
std::thread monitor_thread_;
|
||||
std::atomic<bool> should_stop_{false};
|
||||
std::atomic<bool> stop_requested_{false}; // a stop was asked for: the next exit is not a crash
|
||||
std::set<std::string> debug_categories_;
|
||||
int max_connections_ = 0; // 0 = daemon default
|
||||
bool stratum_enabled_ = false; // -stratum: host a RandomX pool (v1.3.0+; older daemons ignore it)
|
||||
|
||||
Reference in New Issue
Block a user