diff --git a/src/app.cpp b/src/app.cpp index a551ca9..5d29a85 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -5378,105 +5378,231 @@ bool App::startEmbeddedDaemon() return daemon_controller_->start(settings_.get()); } -void App::stopEmbeddedDaemon() +// Drives one stop session for DaemonController::executeDaemonStop: a dedicated RPC client for "stop" (not +// rpc_, which another thread may hold, and whose getinfo probe needs the daemon's cs_main), the owned +// process's exit / signal / CPU, and the shutdown screen's choices. +class AppStopRuntime final : public daemon::DaemonController::StopRuntime { +public: + using DC = daemon::DaemonController; + + AppStopRuntime(App& app, daemon::EmbeddedDaemon* owned, std::vector credentials) + : app_(app), owned_(owned), creds_(std::move(credentials)), t0_(std::chrono::steady_clock::now()) + { + std::lock_guard lk(app_.stop_client_mutex_); + app_.active_stop_client_ = &client_; + } + ~AppStopRuntime() override + { + std::lock_guard lk(app_.stop_client_mutex_); + app_.active_stop_client_ = nullptr; + } + + double nowSec() override + { + return std::chrono::duration(std::chrono::steady_clock::now() - t0_).count(); + } + bool processAlive() override + { + if (owned_) return owned_->pollOwnedProcess(); + // Not ours: we can only watch for it to go (port and process), never wait on or signal it. + return daemon::EmbeddedDaemon::isRpcPortInUse() || daemon::EmbeddedDaemon::isDaemonProcessRunning(); + } + daemon::StopSendResult trySendStop() override + { + if (creds_.empty()) return daemon::StopSendResult::Other; + if (!clientOpen_) { + const auto& c = creds_[credIndex_]; + clientOpen_ = client_.open(c.host, c.port, c.rpcuser, c.rpcpassword, c.use_tls); + if (!clientOpen_) return daemon::StopSendResult::Other; + } + const rpc::RpcFailureKind kind = client_.trySendStop(2); + const daemon::StopSendResult r = daemon::stopSendResultFromKind(kind); + ++attempts_; + if (attempts_ == 1 || r != lastLogged_) { // log changes, not every retry + DEBUG_LOGF("[stop] attempt %d at %.1fs: %s (%s)\n", attempts_, nowSec(), + daemon::stopSendResultName(r), rpc::rpcFailureKindName(kind)); + lastLogged_ = r; + } + return r; + } + bool nextCredentials() override + { + if (credIndex_ + 1 >= creds_.size()) return false; + ++credIndex_; + clientOpen_ = false; + DEBUG_LOGF("[stop] HTTP 401 — retrying with the next credential source (%zu of %zu)\n", + credIndex_ + 1, creds_.size()); + return true; + } + bool sendGracefulSignal() override { return owned_ && owned_->signalGracefulStop(); } + double cpuSeconds() override + { + usage_ = owned_ ? owned_->ownedResourceUsage() : daemon::EmbeddedDaemon::ResourceUsage{}; + return usage_.cpuSeconds; + } + double ioBytes() override { return usage_.ioBytes; } // sampled with cpuSeconds() + bool rebuildActive() override { return false; } // the witness-rebuild tracker is not wired yet + daemon::StopUserChoice userChoice() override + { + return static_cast(app_.stop_user_choice_.load()); + } + void offerForceKill(bool offered) override + { + DEBUG_LOGF("[stop] %s\n", offered ? "no CPU/I-O progress for a long time: offering Kill" + : "progress resumed: Kill offer withdrawn"); + app_.stop_kill_offered_ = offered; + } + void forceKill() override + { + DEBUG_LOGF("[stop] user chose Kill after %.0fs\n", nowSec()); + if (owned_) owned_->forceKillOwnedProcess(); + } + void setStatus(const DC::StopStatus& s) override + { + if (s.delivered && !loggedDelivered_) { + DEBUG_LOGF("[stop] stop delivered at %.1fs (%s)\n", s.elapsedSec, + s.signalled ? "SIGTERM to the process group" : daemon::stopSendResultName(s.lastSend)); + loggedDelivered_ = true; + } + std::string text = TR(s.delivered ? "shutdown_stop_delivered" + : s.heard ? "shutdown_stop_queued" + : "shutdown_stop_asking"); + if (s.cpuPercent >= 0) { + char buf[64]; + snprintf(buf, sizeof(buf), TR("shutdown_stop_cpu_suffix"), static_cast(s.cpuPercent + 0.5)); + text += buf; + } + app_.shutdown_status_ = text; + app_.stop_idle_minutes_ = static_cast(s.idleSec / 60.0); + } + +private: + App& app_; + daemon::EmbeddedDaemon* owned_ = nullptr; + std::vector creds_; + size_t credIndex_ = 0; + rpc::RPCClient client_; + bool clientOpen_ = false; + int attempts_ = 0; + daemon::StopSendResult lastLogged_ = daemon::StopSendResult::Other; + bool loggedDelivered_ = false; + daemon::EmbeddedDaemon::ResourceUsage usage_; + std::chrono::steady_clock::time_point t0_; +}; + +class AppStopTaskContext final : public daemon::DaemonController::LifecycleTaskContext { +public: + AppStopTaskContext(App& app, bool watchShutdown) : app_(app), watchShutdown_(watchShutdown) {} + bool cancelled() const override { return false; } + bool shuttingDown() const override { return watchShutdown_ && app_.shutting_down_.load(); } + void sleepForMs(int ms) override { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); } + +private: + App& app_; + bool watchShutdown_ = false; +}; + +void App::chooseStopAction(daemon::StopUserChoice choice) { - if (!daemon_controller_) return; + stop_user_choice_ = static_cast(choice); + if (choice == daemon::StopUserChoice::Detach) { + // End a pending stop attempt now rather than after its timeout. + std::lock_guard lk(stop_client_mutex_); + if (active_stop_client_) active_stop_client_->requestAbort(); + } +} + +daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode) +{ + if (!daemon_controller_) return daemon::StopOutcome::Exited; auto shutdownDecision = daemon_controller_->shutdownDecision( false, settings_ && settings_->getStopExternalDaemon()); if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) { DEBUG_LOGF("stopEmbeddedDaemon: %s — skipping\n", shutdownDecision.logReason); - return; + return daemon::StopOutcome::StillRunning; } - // Do we hold a live process handle for this node? Capture it BEFORE stop() reaps the pid. If owned, - // daemon_controller_->stop() below BLOCKS until the process actually exits. If NOT owned (external / - // adopted / direct-connected / a lock-gated start that never spawned), stop() returns at once on every - // platform — it only ever waits on (or kills) a process this app launched, never one found by name — - // so we can only ask it over RPC and then watch for it to disappear (handled after the stop() call). - const bool owned = daemon_controller_->isRunning(); + // One stop at a time: a second caller (say the shutdown thread while a restart task is stopping the + // node) waits for the first. Background stops let go as soon as the app starts quitting, so the + // shutdown never waits long behind one. + std::lock_guard single(stop_mutex_); + daemon::EmbeddedDaemon* d = daemon_controller_->daemon(); + // beginStop() also marks the stop as requested (an early exit is not a crash) and makes this session + // the only reaper. Only a process we launched and still hold is "owned": a lock-gated start, an + // external node and a node we already let go of are not, and are never waited on, signalled or killed. + const bool owned = d && d->beginStop(); + if (d && !owned) d->endStop(/*keepWatching=*/false); - // Any exit from here on is the stop we are asking for, not a crash — even one that lands before - // daemon_controller_->stop() below gets to reap it (crash_count_ drives auto-restart / give-up). - daemon_controller_->markStopRequested(); + if (!owned && !daemon::EmbeddedDaemon::isRpcPortInUse()) { + // Nothing listening for a stop: either it is gone, or it is another dragonxd we can't talk to (one + // still finishing its own shutdown trips the start() lock gate this way). Leave it be. + const bool running = daemon::EmbeddedDaemon::isDaemonProcessRunning(); + DEBUG_LOGF("stopEmbeddedDaemon: no node of ours and no RPC listener%s\n", + running ? " — another dragonxd is running; leaving it alone" : ""); + return running ? daemon::StopOutcome::StillRunning : daemon::StopOutcome::Exited; + } - // Send RPC "stop" command — this is the graceful path that lets the - // daemon flush state, save block indexes, close sockets, etc. - bool stop_sent = false; - - // Try the existing RPC connection first. beginShutdown() called rpc_->requestAbort() to unblock any - // in-flight request — but that sticky abort (cleared only by connect()) would make THIS "stop" self- - // abort mid-flight (CURLE_ABORTED_BY_CALLBACK). The old async rpc_->stop() swallowed that error yet - // set stop_sent=true, so the daemon never heard "stop" and the working temp-connection fallback was - // skipped — which is why "Stop external daemon" left an external dragonxd running (it only died via - // the 20s force-kill). Clear the abort and send SYNCHRONOUSLY so real success is surfaced. - if (rpc_ && rpc_->isConnected()) { - DEBUG_LOGF("Sending stop command via existing RPC connection...\n"); - rpc_->clearAbort(); - if (sendStopCommandSafely(*rpc_, "existing connection stop")) - stop_sent = true; + // Credentials for the stop: our own node's conf (and, if those are stale, our last working + // connection, then the .cookie); a node we don't own, the exact credentials we connected with. + std::vector creds; + const rpc::ConnectionConfig detected = rpc::Connection::autoDetectConfig(); + const bool haveSaved = !saved_config_.rpcuser.empty() || !saved_config_.rpcpassword.empty(); + if (owned) { + creds.push_back(detected); + if (haveSaved && (saved_config_.rpcuser != detected.rpcuser || + saved_config_.rpcpassword != detected.rpcpassword || saved_config_.port != detected.port)) + creds.push_back(saved_config_); + rpc::ConnectionConfig cookie; + if (rpc::Connection::buildCookieAuthConfig(detected, cookie)) creds.push_back(cookie); + } else { + creds.push_back(haveSaved ? saved_config_ : detected); } - - // If the main connection wasn't established (e.g. daemon was still - // starting up when user closed the window), create a temporary - // RPC connection just to send the stop command. - if (!stop_sent) { - DEBUG_LOGF("Main RPC not connected — creating temporary connection for stop...\n"); - auto config = rpc::Connection::autoDetectConfig(); - if (!config.rpcuser.empty() && !config.rpcpassword.empty()) { - auto tmp_rpc = std::make_unique(); - // Use a short timeout — if daemon isn't listening yet, don't block - if (tmp_rpc->connect(config.host, config.port, - config.rpcuser, config.rpcpassword, - config.use_tls)) { - DEBUG_LOGF("Temporary RPC connected, sending stop...\n"); - if (sendStopCommandSafely(*tmp_rpc, "Temporary daemon stop")) { - stop_sent = true; - } - tmp_rpc->disconnect(); - } else { - DEBUG_LOGF("Could not establish temporary RPC connection\n"); - } - } else { - DEBUG_LOGF("No RPC credentials available (DRAGONX.conf missing?)\n"); - } - } - - if (stop_sent) { - DEBUG_LOGF("Waiting for daemon to flush block index and shut down...\n"); - shutdown_status_ = "Waiting for daemon to flush block index..."; - // Give the daemon time to flush LevelDB to disk before we - // escalate to SIGTERM. On macOS/APFS, LevelDB compaction + - // fsync can take 15-20s on a large chain. The stop() method - // will wait this long for a *natural* exit (via the RPC stop - // we already sent) before falling back to SIGTERM. - } - - shutdown_status_ = "Waiting for dragonxd process to exit..."; - // 20s grace period for the RPC "stop" to complete (LevelDB flush). - // Only after that does stop() escalate to SIGTERM, then SIGKILL. - daemon_controller_->stop(20000); - // EXTERNAL / adopted node during app shutdown: we hold no process handle, so the stop() above returned - // immediately (it can only wait on a node WE spawned). But the user turned on "Stop external daemon", - // so keep the window on the shutdown screen and poll until the node is actually gone — surfacing a live - // status so they can SEE it stop — rather than closing while it's still flushing. Bounded ~120s (a - // graceful full-node shutdown can flush LevelDB for 60-90s). Scoped to real shutdown; the shutdown - // screen's Force Quit stays available and flips shutdown_complete_, which breaks us out at once. - if (stop_sent && !owned && shutting_down_) { - auto stillUp = []() { - return daemon::EmbeddedDaemon::isRpcPortInUse() || daemon::EmbeddedDaemon::isDaemonProcessRunning(); - }; - // Set the phase text ONCE, then just poll — the shutdown screen already renders a live "N seconds" - // elapsed counter on the UI thread, so the user still sees time passing. (shutdown_status_ is now a - // GuardedStatus, so per-iteration writes would be race-safe; the single write is just a UX choice.) - shutdown_status_ = "Waiting for the external node to stop..."; - for (int i = 0; i < 1200 && stillUp() && !shutdown_complete_; ++i) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - shutdown_status_ = stillUp() ? "External node still stopping — closing anyway..." - : "External node stopped"; - DEBUG_LOGF("stopEmbeddedDaemon: external node %s\n", stillUp() ? "still up (timed out)" : "confirmed stopped"); + daemon::DaemonController::StopPolicy policy; + policy.owned = owned; +#ifndef _WIN32 + policy.gracefulSignal = owned; // dragonxd treats SIGTERM like stop; Windows has no such channel +#endif + switch (mode) { + case StopMode::AppShutdown: + policy.userChoices = true; + policy.maxWaitSec = owned ? 0.0 : 120.0; // a node we don't own: watch it go for up to 2 min + break; + case StopMode::Background: + policy.maxWaitSec = 120.0; + policy.abandonOnShutdown = true; + stop_user_choice_ = 0; + break; + case StopMode::Fallback: + policy.maxWaitSec = 20.0; + stop_user_choice_ = 0; + break; } + + DEBUG_LOGF("stopEmbeddedDaemon: %s node, %s\n", owned ? "our own" : "an external", + mode == StopMode::AppShutdown ? "quitting (no time limit; the user may leave it running)" + : mode == StopMode::Background ? "background (up to 120s, never killed)" + : "fallback (up to 20s, never killed)"); + stop_kill_offered_ = false; + stop_session_active_ = true; + daemon::StopOutcome outcome; + { + AppStopRuntime runtime(*this, owned ? d : nullptr, std::move(creds)); + AppStopTaskContext task(*this, policy.abandonOnShutdown); + outcome = daemon::DaemonController::executeDaemonStop(policy, runtime, task); + } + stop_session_active_ = false; + stop_kill_offered_ = false; + // Still running after a background wait: keep watching it (its exit is booked as a clean stop). Left + // running on purpose ("Leave running"), exited or killed: let go of it. + if (owned) d->endStop(/*keepWatching=*/outcome == daemon::StopOutcome::StillRunning); + DEBUG_LOGF("stopEmbeddedDaemon: %s\n", + outcome == daemon::StopOutcome::Exited ? "the node exited" + : outcome == daemon::StopOutcome::Detached ? "left running at the user's request" + : outcome == daemon::StopOutcome::Killed ? "killed at the user's request" + : "still running (stopped waiting; never killed)"); + return outcome; } bool App::isEmbeddedDaemonRunning() const @@ -5493,10 +5619,10 @@ bool App::stopDaemonForWalletSwitch() // not start the node. Gate on the handle instead. const bool owned = daemon_controller_ && isEmbeddedDaemonRunning(); if (owned) { - // We spawned it — stopEmbeddedDaemon() BLOCKS for full process exit (handle wait + SIGTERM/SIGKILL - // escalation), so the datadir lock + RPC port are released by the time it returns; a short port - // check just confirms. - stopEmbeddedDaemon(); + // We spawned it — stopEmbeddedDaemon() waits for the process to exit (up to 120s, never killing + // it), so on Exited the datadir lock + RPC port are released; a short port check just confirms. + // Still running means the switch must not go ahead: the old node holds the wallet and the lock. + if (stopEmbeddedDaemon() != daemon::StopOutcome::Exited) return shutting_down_; for (int i = 0; i < 200 && daemon::EmbeddedDaemon::isRpcPortInUse() && !shutting_down_; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(100)); return shutting_down_ || !daemon::EmbeddedDaemon::isRpcPortInUse(); @@ -5808,6 +5934,8 @@ void App::beginShutdown() shutting_down_ = true; quit_requested_ = true; shutdown_timer_ = 0.0f; + stop_user_choice_ = 0; + stop_kill_offered_ = false; shutdown_start_time_ = std::chrono::steady_clock::now(); async_tasks_.cancelAll(); // The seed-migration adopt task drives daemon stop/start and swaps wallet.dat; let it finish @@ -5883,8 +6011,9 @@ void App::beginShutdown() shutdown_thread_ = std::thread([this]() { DEBUG_LOGF("shutdown thread: calling stopEmbeddedDaemon()\n"); shutdown_status_ = TR("appx_sending_stop_command_to_daemon"); - // Send RPC stop command - stopEmbeddedDaemon(); + // Deliver stop and wait for the node to exit: no time limit, never killed unasked. The shutdown + // screen offers "Leave running" (and "Kill" if the node goes idle), which end the wait at once. + stopEmbeddedDaemon(StopMode::AppShutdown); DEBUG_LOGF("shutdown thread: daemon stopped, disconnecting RPC\n"); shutdown_status_ = TR("appx_cleaning_up"); @@ -6055,17 +6184,10 @@ void App::renderShutdownScreen() }; shutdown_timer_ += ImGui::GetIO().DeltaTime; - // Track how long the status text has been unchanged. A normal shutdown keeps updating the status - // ("Stopping pool miner…" -> "Flushing…" -> "Cleaning up…"); a genuine hang stalls it. Offer Force - // Quit only once the status has STALLED (likely hung), not on a bare timer, with a hard ceiling so - // it's never impossible to escape. - static std::string s_lastShutStatus; - static float s_shutStallTimer = 0.0f; const std::string curShut = shutdown_status_.get(); // one consistent snapshot per frame (M-05) - if (curShut != s_lastShutStatus) { s_lastShutStatus = curShut; s_shutStallTimer = 0.0f; } - else s_shutStallTimer += ImGui::GetIO().DeltaTime; - const bool shutdownStalled = s_shutStallTimer >= 8.0f; - const bool allowForceQuit = shutdownStalled || shutdown_timer_ >= 20.0f; + // After a few seconds, offer to close the wallet and leave the node running: stopping can take as long + // as the node needs (a witness rebuild can hold it for an hour), and it is never killed unasked. + const bool allowChoices = shutdown_timer_ >= 3.0f; // Use the main viewport so the overlay covers the primary window ImGuiViewport* vp = ImGui::GetMainViewport(); @@ -6086,9 +6208,8 @@ void App::renderShutdownScreen() ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoScrollbar | ImGuiWindowFlags_NoSavedSettings; - // Allow input only once Force Quit is offered (status stalled / hard ceiling), so the screen stays - // click-through while a normal shutdown is progressing. - if (!allowForceQuit) + // Allow input only once the choices are offered, so the screen stays click-through for a quick exit. + if (!allowChoices) shutdownFlags |= ImGuiWindowFlags_NoInputs | ImGuiWindowFlags_NoNav; ImGui::Begin("##ShutdownOverlay", nullptr, shutdownFlags); @@ -6189,51 +6310,64 @@ void App::renderShutdownScreen() } // ------------------------------------------------------------------- - // 4b. Force Quit button — appears once the shutdown appears stalled (or a hard-ceiling backstop) + // 4b. "Leave running" (always, after a few seconds) and "Kill" (only once the node looks stuck) // ------------------------------------------------------------------- - if (allowForceQuit) { + if (allowChoices) { ImGui::Spacing(); ImGui::Spacing(); - // State-aware caution: while the status is a daemon flush/exit step, force-quitting risks the - // chainstate; say so instead of a bare button. - if (shutdownStalled && !curShut.empty()) { - std::string stalledMsg = std::string(TR("appx_still_status_prefix")) + curShut + - TR("appx_still_status_suffix"); - ImVec2 ms = ImGui::CalcTextSize(stalledMsg.c_str()); - ImGui::SetCursorPosX(cx - ms.x * 0.5f); - ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(Warning())); - ImGui::TextUnformatted(stalledMsg.c_str()); - ImGui::PopStyleColor(); - ImGui::Spacing(); - } - const char* forceLabel = TR("force_quit"); - ImVec2 btnSize(ImGui::CalcTextSize(forceLabel).x + 32.0f * dp, 0); - ImGui::SetCursorPosX(cx - btnSize.x * 0.5f); - ImGui::PushStyleColor(ImGuiCol_Button, ImVec4(0.6f, 0.15f, 0.15f, 0.9f)); - ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.75f, 0.2f, 0.2f, 1.0f)); - ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImVec4(0.5f, 0.1f, 0.1f, 1.0f)); - if (ui::material::TactileButton(forceLabel, btnSize)) { - force_quit_confirm_ = true; + const char* leaveLabel = TR("shutdown_leave_running"); + ImVec2 leaveSize(ImGui::CalcTextSize(leaveLabel).x + 32.0f * dp, 0); + ImGui::SetCursorPosX(cx - leaveSize.x * 0.5f); + if (ui::material::TactileButton(leaveLabel, leaveSize)) { + DEBUG_LOGF("Shutdown: user chose to leave the node running after %.0fs\n", shutdown_timer_); + chooseStopAction(daemon::StopUserChoice::Detach); + // No stop session (yet): nothing to wait for, so close now. A session that starts later sees + // the choice and returns at once. + if (!stop_session_active_) shutdown_complete_ = true; } if (ImGui::IsItemHovered()) { - ui::material::Tooltip("%s", TR("force_quit_warning")); + ui::material::Tooltip("%s", TR("shutdown_leave_running_tip")); + } + + if (stop_kill_offered_) { + ImGui::Spacing(); + char idleMsg[160]; + snprintf(idleMsg, sizeof(idleMsg), TR("shutdown_stop_idle"), stop_idle_minutes_.load()); + ImVec2 ms = ImGui::CalcTextSize(idleMsg); + ImGui::SetCursorPosX(cx - ms.x * 0.5f); + ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(Warning())); + ImGui::TextUnformatted(idleMsg); + ImGui::PopStyleColor(); + ImGui::Spacing(); + const char* killLabel = TR("shutdown_kill_node"); + ImVec2 killSize(ImGui::CalcTextSize(killLabel).x + 32.0f * dp, 0); + ImGui::SetCursorPosX(cx - killSize.x * 0.5f); + ImGui::PushStyleColor(ImGuiCol_Button, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 210))); + ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImGui::ColorConvertU32ToFloat4(Error())); + ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 160))); + if (ui::material::TactileButton(killLabel, killSize)) { + stop_kill_confirm_ = true; + } + ImGui::PopStyleColor(3); + if (ImGui::IsItemHovered()) { + ui::material::Tooltip("%s", TR("shutdown_kill_node_tip")); + } } - ImGui::PopStyleColor(3); } - // Force Quit confirmation popup - if (force_quit_confirm_) { - ImGui::OpenPopup("##ForceQuitConfirm"); - force_quit_confirm_ = false; + // Kill confirmation + if (stop_kill_confirm_) { + ImGui::OpenPopup("##KillNodeConfirm"); + stop_kill_confirm_ = false; } ImVec2 popupCenter(wp.x + vp_size.x * 0.5f, wp.y + vp_size.y * 0.5f); ImGui::SetNextWindowPos(popupCenter, ImGuiCond_Appearing, ImVec2(0.5f, 0.5f)); - if (ImGui::BeginPopupModal("##ForceQuitConfirm", nullptr, ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoTitleBar)) { + if (ImGui::BeginPopupModal("##KillNodeConfirm", nullptr, ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoTitleBar)) { ImGui::PushFont(Type().subtitle1()); - ImGui::TextUnformatted(TR("force_quit_confirm_title")); + ImGui::TextUnformatted(TR("shutdown_kill_confirm_title")); ImGui::PopFont(); ImGui::Spacing(); - ImGui::TextUnformatted(TR("force_quit_confirm_msg")); + ImGui::TextUnformatted(TR("shutdown_kill_confirm_msg")); ImGui::Spacing(); ImGui::Spacing(); @@ -6249,9 +6383,9 @@ void App::renderShutdownScreen() ImGui::PushStyleColor(ImGuiCol_Button, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 210))); ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImGui::ColorConvertU32ToFloat4(Error())); ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImGui::ColorConvertU32ToFloat4(WithAlpha(Error(), 160))); - if (ui::material::TactileButton(TR("force_quit_yes"), ImVec2(btnW, 0))) { - DEBUG_LOGF("Force quit confirmed by user after %.0fs\n", shutdown_timer_); - shutdown_complete_ = true; + if (ui::material::TactileButton(TR("shutdown_kill_confirm_yes"), ImVec2(btnW, 0))) { + DEBUG_LOGF("Shutdown: user confirmed Kill after %.0fs\n", shutdown_timer_); + chooseStopAction(daemon::StopUserChoice::Kill); ImGui::CloseCurrentPopup(); } ImGui::PopStyleColor(3); @@ -6767,7 +6901,7 @@ void App::shutdown() settings_->save(); } if (daemon_controller_) { - stopEmbeddedDaemon(); + stopEmbeddedDaemon(StopMode::Fallback); // on the UI thread: bounded, never killed } if (rpc_) { rpc_->disconnect(); diff --git a/src/app.h b/src/app.h index 6b2b289..507b702 100644 --- a/src/app.h +++ b/src/app.h @@ -21,6 +21,7 @@ #include "data/wallet_index.h" #include "data/wallet_state.h" #include "rpc/connection.h" +#include "daemon/stop_delivery.h" #include "services/network_refresh_service.h" #include "services/wallet_security_controller.h" #include "services/wallet_security_workflow.h" @@ -480,7 +481,13 @@ public: // Embedded daemon control bool startEmbeddedDaemon(); - void stopEmbeddedDaemon(); + // How long stopEmbeddedDaemon() may wait, and who decides. It never kills the node on its own. + enum class StopMode { + AppShutdown, // quitting: no time limit; the shutdown screen offers "Leave running" (and "Kill" when stuck) + Background, // restart/rescan/switch/...: up to 120s, and let go as soon as the app starts quitting + Fallback, // the synchronous shutdown fallback on the UI thread: up to 20s + }; + daemon::StopOutcome stopEmbeddedDaemon(StopMode mode = StopMode::Background); // Stop the node specifically for a wallet switch, which MUST free the RPC port to relaunch on // -wallet=. Unlike stopEmbeddedDaemon() (whose DisconnectOnly policy leaves an adopted/external // daemon running), this sends a graceful RPC "stop" — using our own connection creds, so only our @@ -632,6 +639,8 @@ public: private: friend class AppDaemonLifecycleRuntime; + friend class AppStopRuntime; + friend class AppStopTaskContext; friend class AppDaemonLifecycleTaskContext; // Global keyboard-shortcut handling, dispatched once per frame from update(). @@ -964,9 +973,19 @@ private: bool shutdown_confirmed_ = false; // user chose to proceed — bypass the guard on re-entry bool shutdown_keep_daemon_override_ = false; // user chose "keep node running" for this shutdown only float shutdown_timer_ = 0.0f; - bool force_quit_confirm_ = false; std::chrono::steady_clock::time_point shutdown_start_time_; + // One stop session at a time (stopEmbeddedDaemon), and the shutdown screen's view of it. + std::mutex stop_mutex_; + std::atomic stop_session_active_{false}; + std::atomic stop_user_choice_{0}; // daemon::StopUserChoice, set by the shutdown screen + std::atomic stop_kill_offered_{false}; // the node looks stuck: the screen may offer "Kill" + std::atomic stop_idle_minutes_{0}; + std::mutex stop_client_mutex_; + rpc::RPCClient* active_stop_client_ = nullptr; // aborted on "Leave running" so a pending attempt ends at once + bool stop_kill_confirm_ = false; + void chooseStopAction(daemon::StopUserChoice choice); + // Daemon restart (e.g. after changing debug log categories) std::atomic daemon_restarting_{false}; diff --git a/src/daemon/daemon_controller.h b/src/daemon/daemon_controller.h index 5a40b8f..ea03c90 100644 --- a/src/daemon/daemon_controller.h +++ b/src/daemon/daemon_controller.h @@ -1,9 +1,11 @@ #pragma once #include "embedded_daemon.h" +#include "stop_delivery.h" #include #include +#include #include #include #include @@ -234,6 +236,215 @@ public: ShutdownDecision shutdownDecision(bool keepDaemonRunning, bool stopExternalDaemon) const; + // ------------------------------------------------------------------------------------------------ + // Stopping the node: deliver "stop" until it lands, then wait for the process to exit. Never kill it + // on our own: a node that is busy (a witness rebuild can hold it for an hour) or flushing would lose + // everything since its last save and replay it on the next start. Only an explicit user "Kill", + // offered once the node has gone idle, ends it by force; "Leave running" lets the wallet close while + // the node finishes on its own. + // ------------------------------------------------------------------------------------------------ + + using StopUserChoice = daemon::StopUserChoice; + using StopOutcome = daemon::StopOutcome; + + struct StopPolicy { + bool owned = true; // we launched it: watch the process, signal it, offer Kill + bool gracefulSignal = false; // POSIX: a group SIGTERM is a stop channel (dragonxd treats it as stop) + double maxWaitSec = 0.0; // stop waiting (never kill) after this long; 0 = no limit + bool userChoices = false; // the shutdown screen offers "Leave running" / "Kill" + bool abandonOnShutdown = false; // stop waiting once the app starts shutting down (background callers) + double idleOfferSec = 600.0; // delivered + no CPU/I-O progress for this long: offer Kill + }; + + // What the stop loop reports for display. + struct StopStatus { + bool delivered = false; // stop accepted (or the node is shutting down, or was signalled) + bool heard = false; // the node has the request (possibly still queued) + bool signalled = false; // a graceful signal (POSIX SIGTERM) went out + double elapsedSec = 0.0; + double cpuPercent = -1.0; // < 0: unknown + double idleSec = 0.0; + bool killOffered = false; + bool rebuildActive = false; + StopSendResult lastSend = StopSendResult::Other; + int attempts = 0; + }; + + class StopRuntime { + public: + virtual ~StopRuntime() = default; + virtual double nowSec() = 0; + virtual bool processAlive() = 0; + virtual StopSendResult trySendStop() = 0; + virtual bool nextCredentials() = 0; // switch to the next stop credentials; false when none left + virtual bool sendGracefulSignal() = 0; // POSIX group SIGTERM; false where there is none (Windows) + virtual double cpuSeconds() = 0; // cumulative CPU of the node; < 0 if unknown + virtual double ioBytes() = 0; // cumulative I/O bytes of the node; < 0 if unknown + virtual bool rebuildActive() = 0; // a witness rebuild is running: never offer Kill + virtual StopUserChoice userChoice() = 0; + virtual void offerForceKill(bool offered) = 0; + virtual void forceKill() = 0; + virtual void setStatus(const StopStatus& status) = 0; + }; + + /** + * "Making progress" = >= 0.05 cores of CPU or >= 1 MB of I/O over the last >= 60s. A flush that + * fsyncs a big wallet is mostly I/O at low CPU; a rebuild is mostly CPU. Idle time counts from the + * first sample. With no readings at all (unknown), it never reports idle, so Kill is never offered. + */ + class StopProgressTracker { + public: + static constexpr double kWindowSec = 60.0; + static constexpr double kMinCores = 0.05; + static constexpr double kMinIoBytes = 1024.0 * 1024.0; + + void add(double t, double cpuSec, double ioBytes) + { + if (cpuSec >= 0 || ioBytes >= 0) measurable_ = true; + if (samples_.empty()) lastProgressAt_ = t; + samples_.push_back({t, cpuSec, ioBytes}); + // Keep the newest sample that is at least a full window old as the comparison base. + while (samples_.size() > 2 && samples_[1].t <= t - kWindowSec) samples_.pop_front(); + const Sample& base = samples_.front(); + const double span = t - base.t; + if (span >= kWindowSec) { + const bool cpuMoved = cpuSec >= 0 && base.cpu >= 0 && (cpuSec - base.cpu) / span >= kMinCores; + const bool ioMoved = ioBytes >= 0 && base.io >= 0 && (ioBytes - base.io) >= kMinIoBytes; + if (cpuMoved || ioMoved) lastProgressAt_ = t; + } + } + double idleSec(double t) const { return measurable_ ? std::max(0.0, t - lastProgressAt_) : 0.0; } + // CPU over the most recent sample interval (100 = one core), for display; < 0 unknown. + double cpuPercent() const + { + if (samples_.size() < 2) return -1.0; + const Sample& a = samples_[samples_.size() - 2]; + const Sample& b = samples_.back(); + if (a.cpu < 0 || b.cpu < 0 || b.t <= a.t) return -1.0; + return std::max(0.0, (b.cpu - a.cpu) / (b.t - a.t) * 100.0); + } + + private: + struct Sample { double t, cpu, io; }; + std::deque samples_; + double lastProgressAt_ = 0.0; + bool measurable_ = false; + }; + + enum class StopWaitDecision { Done, Wait, GiveUp, Detach, OfferForceKill, ForceKill }; + + struct StopWaitSample { + bool processAlive = true; + bool delivered = false; + double elapsedSec = 0.0; + double idleSec = 0.0; + bool rebuildActive = false; + StopUserChoice choice = StopUserChoice::None; + bool killOffered = false; // the Kill offer is on screen right now + bool shuttingDown = false; // the app began shutting down / the task was cancelled + }; + + static inline StopWaitDecision evaluateStopWait(const StopPolicy& p, const StopWaitSample& s) + { + if (!s.processAlive) return StopWaitDecision::Done; + if (p.userChoices && s.choice == StopUserChoice::Detach) return StopWaitDecision::Detach; + if (p.userChoices && p.owned && s.choice == StopUserChoice::Kill && s.killOffered) + return StopWaitDecision::ForceKill; + if (p.abandonOnShutdown && s.shuttingDown) return StopWaitDecision::GiveUp; + if (p.maxWaitSec > 0 && s.elapsedSec >= p.maxWaitSec) return StopWaitDecision::GiveUp; + if (p.userChoices && p.owned && s.delivered && !s.rebuildActive && s.idleSec >= p.idleOfferSec) + return StopWaitDecision::OfferForceKill; + return StopWaitDecision::Wait; + } + + static constexpr int kStopTickMs = 250; + + static inline StopOutcome executeDaemonStop(const StopPolicy& policy, StopRuntime& rt, + LifecycleTaskContext& task) + { + if (!rt.processAlive()) return StopOutcome::Exited; // already gone: not a single RPC call + + const double t0 = rt.nowSec(); + StopDeliveryPolicy delivery(policy.owned); + StopProgressTracker progress; + bool signalled = false; + bool offered = false; + auto withdrawOffer = [&] { + if (offered) { offered = false; rt.offerForceKill(false); } + }; + + for (;;) { + double now = rt.nowSec() - t0; + if (delivery.shouldSend(now)) { + const StopSendResult r = delivery.record(rt.trySendStop(), now); + if (r == StopSendResult::Auth && policy.owned) { + // Our own node refusing our credentials means they're stale (e.g. a regenerated + // .cookie): try the next source; out of sources, the graceful signal below remains. + if (rt.nextCredentials()) delivery.retryNow(rt.nowSec() - t0); + else delivery.giveUp(); + } + // POSIX: a group SIGTERM bypasses the RPC queue altogether (dragonxd's handler sets the same + // flag as stop). Send it as soon as an attempt shows stop didn't get through. + if (policy.gracefulSignal && !signalled && !delivery.delivered() && + r != StopSendResult::Queued && rt.sendGracefulSignal()) { + signalled = true; + delivery.markDelivered(); + } + } + if (!rt.processAlive()) { + withdrawOffer(); + return StopOutcome::Exited; + } + + now = rt.nowSec() - t0; + progress.add(now, rt.cpuSeconds(), rt.ioBytes()); + StopWaitSample s; + s.delivered = delivery.delivered(); + s.elapsedSec = now; + s.idleSec = progress.idleSec(now); + s.rebuildActive = rt.rebuildActive(); + s.choice = rt.userChoice(); + s.killOffered = offered; + s.shuttingDown = task.cancelled() || task.shuttingDown(); + + switch (evaluateStopWait(policy, s)) { + case StopWaitDecision::Done: + withdrawOffer(); + return StopOutcome::Exited; + case StopWaitDecision::Detach: + withdrawOffer(); + return StopOutcome::Detached; + case StopWaitDecision::ForceKill: + withdrawOffer(); + rt.forceKill(); + return StopOutcome::Killed; + case StopWaitDecision::GiveUp: + withdrawOffer(); + return StopOutcome::StillRunning; + case StopWaitDecision::OfferForceKill: + if (!offered) { offered = true; rt.offerForceKill(true); } + break; + case StopWaitDecision::Wait: + withdrawOffer(); // progress resumed (or a rebuild started): the node isn't stuck after all + break; + } + + StopStatus st; + st.delivered = delivery.delivered(); + st.heard = delivery.heard(); + st.signalled = signalled; + st.elapsedSec = now; + st.cpuPercent = progress.cpuPercent(); + st.idleSec = s.idleSec; + st.killOffered = offered; + st.rebuildActive = s.rebuildActive; + st.lastSend = delivery.last(); + st.attempts = delivery.attempts(); + rt.setStatus(st); + task.sleepForMs(kStopTickMs); + } + } + private: std::unique_ptr daemon_; }; diff --git a/src/daemon/embedded_daemon.cpp b/src/daemon/embedded_daemon.cpp index a7a1558..d86df1a 100644 --- a/src/daemon/embedded_daemon.cpp +++ b/src/daemon/embedded_daemon.cpp @@ -948,6 +948,78 @@ bool EmbeddedDaemon::stop(int wait_ms) return exitedOnItsOwn; } +bool EmbeddedDaemon::beginStop() +{ + stop_requested_ = true; + should_stop_ = true; + if (monitor_thread_.joinable()) { + monitor_thread_.join(); + } + if (!isRunning()) return false; // handle-based: our own process only + setState(State::Stopping, "Stopping dragonxd..."); + return true; +} + +bool EmbeddedDaemon::pollOwnedProcess() +{ + drainOutput(); + return isRunning(); +} + +bool EmbeddedDaemon::signalGracefulStop() +{ + return false; // no graceful channel on Windows: the GUI launches dragonxd without a console +} + +void EmbeddedDaemon::forceKillOwnedProcess() +{ + if (process_handle_ == nullptr || !isRunning()) return; + DEBUG_LOGF("Killing dragonxd at the user's request (TerminateProcess on our own handle)\n"); + TerminateProcess(process_handle_, 1); + WaitForSingleObject(process_handle_, 5000); + drainOutput(); +} + +EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const +{ + ResourceUsage usage; + if (process_handle_ == nullptr) return usage; + FILETIME created, exited, kernel, user; + if (GetProcessTimes(process_handle_, &created, &exited, &kernel, &user)) { + auto ticks = [](const FILETIME& ft) { + return (static_cast(ft.dwHighDateTime) << 32) | ft.dwLowDateTime; + }; + usage.cpuSeconds = static_cast(ticks(kernel) + ticks(user)) / 1e7; // 100ns units + } + IO_COUNTERS io; + if (GetProcessIoCounters(process_handle_, &io)) + usage.ioBytes = static_cast(io.ReadTransferCount + io.WriteTransferCount); + return usage; +} + +void EmbeddedDaemon::endStop(bool keepWatching) +{ + if (keepWatching && isRunning()) { + // Still running and still ours: watch it again. Its exit is booked as a clean stop. + should_stop_ = false; + monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this); + return; + } + const bool leftRunning = isRunning(); + if (leftRunning) + DEBUG_LOGF("Leaving dragonxd running; it will finish shutting down on its own\n"); + if (process_handle_ != nullptr) { + CloseHandle(process_handle_); // only our handle: the process carries on + process_handle_ = nullptr; + } + if (stdout_read_ != nullptr) { + CloseHandle(stdout_read_); + stdout_read_ = nullptr; + } + drainOutput(); + setState(State::Stopped, leftRunning ? "dragonxd left running" : "dragonxd stopped"); +} + // Translate Windows NTSTATUS / exit codes to human-readable descriptions static std::string translateWindowsExitCode(DWORD code) { switch (code) { @@ -1396,6 +1468,125 @@ bool EmbeddedDaemon::stop(int wait_ms) return exitedOnItsOwn; } +bool EmbeddedDaemon::beginStop() +{ + stop_requested_ = true; + should_stop_ = true; + // The monitor sees should_stop_ within one 100ms tick. If the child exited in that window the + // monitor reaped it (as a clean stop: stop_requested_ is set) and the pid is now 0. + if (monitor_thread_.joinable()) { + monitor_thread_.join(); + } + if (process_pid_ <= 0) return false; + setState(State::Stopping, "Stopping dragonxd..."); + return true; +} + +bool EmbeddedDaemon::pollOwnedProcess() +{ + const pid_t pid = process_pid_; + if (pid <= 0) return false; + drainOutput(); + int status = 0; + const pid_t r = waitpid(pid, &status, WNOHANG); + if (r == 0 || (r < 0 && errno == EINTR)) return true; + if (r == pid) DEBUG_LOGF("dragonxd exited (%s)\n", describeExitStatus(status).c_str()); + drainOutput(); + process_pid_ = 0; + return false; +} + +bool EmbeddedDaemon::signalGracefulStop() +{ + const pid_t pid = process_pid_; + if (pid <= 0) return false; + // The whole group: with a bash wrapper (embedded_daemon.cpp startProcess) a SIGTERM to the wrapper + // alone would orphan dragonxd. + DEBUG_LOGF("Sending SIGTERM to dragonxd's process group -%d (graceful stop)\n", pid); + return kill(-pid, SIGTERM) == 0; +} + +void EmbeddedDaemon::forceKillOwnedProcess() +{ + const pid_t pid = process_pid_; + if (pid <= 0) return; + DEBUG_LOGF("Killing dragonxd (group -%d) at the user's request\n", pid); + kill(-pid, SIGKILL); + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {} + drainOutput(); + process_pid_ = 0; +} + +EmbeddedDaemon::ResourceUsage EmbeddedDaemon::ownedResourceUsage() const +{ + ResourceUsage usage; +#ifdef __APPLE__ + return usage; // no sampler yet (would need proc_pid_rusage): never "idle", so Kill is never offered +#else + const pid_t pgid = process_pid_; + if (pgid <= 0) return usage; + // The tracked pid may be a bash wrapper that itself uses ~0 CPU: sum the whole process group. + unsigned long long ticks = 0; + double io = 0.0; + bool anyCpu = false, anyIo = false; + std::error_code ec; + fs::directory_iterator it("/proc", ec), end; + for (; !ec && it != end; it.increment(ec)) { + const std::string name = it->path().filename().string(); + if (name.empty() || name[0] < '0' || name[0] > '9') continue; + std::ifstream statFile((it->path() / "stat").string()); + std::string line; + long pgrp = 0; + unsigned long long t = 0; + if (!statFile || !std::getline(statFile, line) || !parseProcStatLine(line, pgrp, t) || pgrp != pgid) + continue; + ticks += t; + anyCpu = true; + std::ifstream ioFile((it->path() / "io").string()); + std::string key; + unsigned long long value = 0; + while (ioFile >> key >> value) { + if (key == "read_bytes:" || key == "write_bytes:") { + io += static_cast(value); + anyIo = true; + } + } + } + const long hz = sysconf(_SC_CLK_TCK); + if (anyCpu && hz > 0) usage.cpuSeconds = static_cast(ticks) / static_cast(hz); + if (anyIo) usage.ioBytes = io; + return usage; +#endif +} + +void EmbeddedDaemon::endStop(bool keepWatching) +{ + const pid_t pid = process_pid_; + if (pid > 0 && keepWatching) { + // Still running and still ours: watch it again. Its exit is booked as a clean stop. + should_stop_ = false; + monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this); + return; + } + if (pid > 0) { + // Leave it running. Close our end of its output pipe: its further writes fail with EPIPE, which + // dragonxd ignores (it ignores SIGPIPE), instead of blocking once a pipe nobody drains is full. + // A detached waiter reaps it when it exits, if this process is still around then. + DEBUG_LOGF("Leaving dragonxd (pid %d) running; it will finish shutting down on its own\n", pid); + std::thread([pid] { + int st = 0; + while (waitpid(pid, &st, 0) < 0 && errno == EINTR) {} + }).detach(); + process_pid_ = 0; + } + if (stdout_fd_ >= 0) { + close(stdout_fd_); + stdout_fd_ = -1; + } + setState(State::Stopped, pid > 0 ? "dragonxd left running" : "dragonxd stopped"); +} + void EmbeddedDaemon::monitorProcess() { char buffer[4096]; diff --git a/src/daemon/embedded_daemon.h b/src/daemon/embedded_daemon.h index b00588a..4406bfa 100644 --- a/src/daemon/embedded_daemon.h +++ b/src/daemon/embedded_daemon.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -71,6 +72,47 @@ public: */ void markStopRequested() { stop_requested_ = true; } + // --- A stop session, driven by DaemonController::executeDaemonStop through the app's StopRuntime --- + + /** Marks the stop as requested and stops the monitor, so the session is the only reaper. + * @return true if a process this object launched is still alive; then finish with endStop(). */ + bool beginStop(); + /** Polls our own process (reaping it on POSIX) and drains its output; false once it has exited. */ + bool pollOwnedProcess(); + /** POSIX: SIGTERM our process group (dragonxd treats it like RPC stop). Windows has no such channel. */ + bool signalGracefulStop(); + /** Ends our own process by force (SIGKILL / TerminateProcess) and reaps it. Explicit user choice only. */ + void forceKillOwnedProcess(); + struct ResourceUsage { + double cpuSeconds = -1.0; // cumulative; < 0 if unknown + double ioBytes = -1.0; // cumulative storage I/O; < 0 if unknown + }; + /** CPU and I/O used so far by our process (its whole process group on Linux). */ + ResourceUsage ownedResourceUsage() const; + /** Finishes a stop session. The process is gone: release everything. Still running: keepWatching + * resumes the monitor (its exit is still seen, and booked as a clean stop); otherwise let it go — + * the user chose to leave it running. */ + void endStop(bool keepWatching); + + /** + * @brief Parse a /proc//stat line: process group (field 5) and utime+stime ticks (fields 14+15) + * + * comm (field 2) is parenthesised and may itself contain spaces and ')', so fields are read from + * after the LAST ')'. + */ + static bool parseProcStatLine(const std::string& line, long& pgrp, unsigned long long& cpuTicks) + { + const size_t close = line.rfind(')'); + if (close == std::string::npos) return false; + unsigned long long utime = 0, stime = 0; + // state ppid pgrp session tty_nr tpgid flags minflt cminflt majflt cmajflt utime stime + if (std::sscanf(line.c_str() + close + 1, " %*c %*d %ld %*d %*d %*d %*u %*u %*u %*u %*u %llu %llu", + &pgrp, &utime, &stime) != 3) + return false; + cpuTicks = utime + stime; + return true; + } + /** * @brief Check if daemon process is running */ diff --git a/src/daemon/stop_delivery.h b/src/daemon/stop_delivery.h new file mode 100644 index 0000000..f9ab33c --- /dev/null +++ b/src/daemon/stop_delivery.h @@ -0,0 +1,155 @@ +#pragma once + +// Delivering "stop" to dragonxd: what one attempt achieved, and when to send another. Pure and +// header-only so the test target can drive it without the RPC client. +// +// Why retrying matters: dragonxd's HTTP server has 8 workers and a 16-deep queue. While a long +// operation holds cs_main, the GUI's own abandoned polls can fill every slot, and each new request is +// refused at once with "Work queue depth exceeded" until a slot frees. The old code made one or two +// attempts, gated on a getinfo probe that needs cs_main, so stop was never delivered and the node was +// then killed. + +#include "rpc/rpc_failure.h" + +namespace dragonx { +namespace daemon { + +// What the user picked on the shutdown screen while the node was stopping. +enum class StopUserChoice { None, Detach, Kill }; + +// How a stop session ended. The wallet never kills the node unless the user chose Kill. +enum class StopOutcome { + Exited, // the process is gone + StillRunning, // we stopped waiting (time budget, app shutdown, cancellation); never killed + Detached, // the user chose to leave it running + Killed, // the user chose Kill +}; + +inline const char* stopOutcomeName(StopOutcome o) +{ + switch (o) { + case StopOutcome::Exited: return "Exited"; + case StopOutcome::StillRunning: return "StillRunning"; + case StopOutcome::Detached: return "Detached"; + case StopOutcome::Killed: return "Killed"; + } + return "Exited"; +} + +enum class StopSendResult { + Accepted, // HTTP 200: the daemon ran stop + Queued, // reply timeout: it reached the daemon and waits for a worker; it will run + QueueFull, // "Work queue depth exceeded": refused before queueing (holds no slot) + NotListening, // refused / connect timeout: RPC isn't bound (not up yet, or already unbound) + ShuttingDown, // 503 / "Unhandled request": the daemon is already shutting down + Auth, // 401: our credentials are wrong or stale + Other, +}; + +inline const char* stopSendResultName(StopSendResult r) +{ + switch (r) { + case StopSendResult::Accepted: return "Accepted"; + case StopSendResult::Queued: return "Queued"; + case StopSendResult::QueueFull: return "QueueFull"; + case StopSendResult::NotListening: return "NotListening"; + case StopSendResult::ShuttingDown: return "ShuttingDown"; + case StopSendResult::Auth: return "Auth"; + case StopSendResult::Other: return "Other"; + } + return "Other"; +} + +// What a stop call's failure kind (RpcFailureKind::None = HTTP 200) means for delivery. +inline StopSendResult stopSendResultFromKind(rpc::RpcFailureKind kind) +{ + using K = rpc::RpcFailureKind; + switch (kind) { + case K::None: return StopSendResult::Accepted; + case K::ReplyTimeout: return StopSendResult::Queued; + case K::QueueFull: return StopSendResult::QueueFull; + case K::NotListening: + case K::ConnectTimeout: return StopSendResult::NotListening; + case K::ShuttingDown: + case K::Unhandled: return StopSendResult::ShuttingDown; + case K::Auth: return StopSendResult::Auth; + default: return StopSendResult::Other; + } +} + +/** + * When to (re)send stop. Times are seconds on the caller's clock. + * + * - Accepted / ShuttingDown: delivered, never resend. + * - Queued: the daemon has it. Resend at most every 30s: each resend takes another queue slot, and + * concurrent stops race on the miner threads (stop's GenerateBitcoins(false)). + * - QueueFull / NotListening / Other: not delivered, cheap to retry: again after 1.5s. + * - NotListening AFTER the daemon had heard us means it unbound its listener to shut down: delivered. + * - Auth: give up for a node we don't own. For our own node the caller may switch credentials and + * retry at once (retryNow), or give up (giveUp). + */ +class StopDeliveryPolicy { +public: + static constexpr double kRetrySec = 1.5; + static constexpr double kQueuedResendSec = 30.0; + + explicit StopDeliveryPolicy(bool owned) : owned_(owned) {} + + bool shouldSend(double now) const { return !finished_ && now >= nextSendAt_; } + + // Records one attempt; returns it as interpreted (a refusal after we were heard reads as ShuttingDown). + StopSendResult record(StopSendResult r, double now) + { + ++attempts_; + if (r == StopSendResult::NotListening && heard_) r = StopSendResult::ShuttingDown; + switch (r) { + case StopSendResult::Accepted: + heard_ = true; + markDelivered(); + break; + case StopSendResult::ShuttingDown: + markDelivered(); + break; + case StopSendResult::Queued: + heard_ = true; + nextSendAt_ = now + kQueuedResendSec; + break; + case StopSendResult::Auth: + authRejected_ = true; + if (!owned_) finished_ = true; + nextSendAt_ = now + kRetrySec; + break; + case StopSendResult::QueueFull: + case StopSendResult::NotListening: + case StopSendResult::Other: + nextSendAt_ = now + kRetrySec; + break; + } + last_ = r; + return r; + } + + void markDelivered() { delivered_ = finished_ = true; } // also for a graceful signal (POSIX SIGTERM) + void retryNow(double now) { nextSendAt_ = now; } + void giveUp() { finished_ = true; } + + bool delivered() const { return delivered_; } // the daemon is stopping (or was signalled) + bool heard() const { return heard_; } // it has (or ran) our request + bool finished() const { return finished_; } // no more attempts + bool authRejected() const { return authRejected_; } + int attempts() const { return attempts_; } + StopSendResult last() const { return last_; } + +private: + bool owned_ = true; + bool delivered_ = false; + bool heard_ = false; + bool finished_ = false; + bool authRejected_ = false; + int attempts_ = 0; + double nextSendAt_ = 0.0; + StopSendResult last_ = StopSendResult::Other; +}; + +} // namespace daemon +} // namespace dragonx diff --git a/src/main.cpp b/src/main.cpp index 7143d2d..f182a86 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2147,10 +2147,9 @@ int main(int argc, char* argv[]) // Watchdog: if cleanup takes too long the process lingers without a // window, showing up as a "Background Service" in Task Manager. - // Force-exit after timeout — all critical state (settings, daemon - // stop) was handled in beginShutdown(). - // Allow enough time for the daemon to shut down gracefully (up to - // 10s in stop()) plus RPC disconnect overhead. + // Force-exit after timeout. The daemon stop is already over by the time the render loop exits: the + // node has exited, or the user chose to leave it running (it carries on as its own process), so this + // only bounds the remaining thread joins and RPC disconnects — it never cuts a node's shutdown short. std::thread([]() { std::this_thread::sleep_for(std::chrono::seconds(15)); fflush(stdout); diff --git a/src/rpc/rpc_client.cpp b/src/rpc/rpc_client.cpp index e0e22e0..bbbcf40 100644 --- a/src/rpc/rpc_client.cpp +++ b/src/rpc/rpc_client.cpp @@ -163,6 +163,17 @@ bool RPCClient::connect(const std::string& host, const std::string& port, bool useTls) { std::lock_guard lk(curl_mutex_); + if (!open(host, port, user, password, useTls)) return false; + return probeConnection(host); +} + +bool RPCClient::open(const std::string& host, const std::string& port, + const std::string& user, const std::string& password, + bool useTls) +{ + std::lock_guard lk(curl_mutex_); + connected_ = false; // only a successful probe (connect()) sets these + warming_up_ = false; host_ = host; port_ = port; last_connect_info_ = json(); @@ -218,7 +229,11 @@ bool RPCClient::connect(const std::string& host, const std::string& port, // build defaults. Harmless on the localhost http:// case; essential for a remote https daemon. curl_easy_setopt(impl_->curl, CURLOPT_SSL_VERIFYPEER, 1L); curl_easy_setopt(impl_->curl, CURLOPT_SSL_VERIFYHOST, 2L); + return true; +} +bool RPCClient::probeConnection(const std::string& host) +{ // Test connection with getinfo. Use a SHORT timeout for the probe on localhost: a healthy // local daemon answers in milliseconds and a warming one returns -28 just as fast, so a long // hang means a wedged/loading occupant — no point blocking the full 30s before we retry and @@ -286,6 +301,17 @@ void RPCClient::logConnectFailure(const std::string& msg) DEBUG_LOGF("Connection failed %s (repeated %u times)\n", line.c_str(), connect_error_repeats_); } +RpcFailureKind RPCClient::trySendStop(long timeoutSec) +{ + try { + TraceScope trace("Daemon lifecycle / Stop command"); + call("stop", json::array(), timeoutSec); + return RpcFailureKind::None; + } catch (const std::exception& e) { + return rpcFailureKindOf(e); + } +} + json RPCClient::getLastConnectInfo() const { std::lock_guard lk(curl_mutex_); diff --git a/src/rpc/rpc_client.h b/src/rpc/rpc_client.h index 809687a..f7552d3 100644 --- a/src/rpc/rpc_client.h +++ b/src/rpc/rpc_client.h @@ -64,6 +64,23 @@ public: const std::string& user, const std::string& password, bool useTls); + /** + * @brief Set up the connection without probing it (connect() = open() + a getinfo probe) + * + * For sending "stop": the getinfo probe needs the daemon's cs_main, so against a busy node it + * times out (holding one of the daemon's few RPC slots) or is refused, and stop never went out. + * isConnected() stays false; call() works once this returns true. + */ + bool open(const std::string& host, const std::string& port, + const std::string& user, const std::string& password, bool useTls); + + /** + * @brief Send "stop" once, with its own timeout; returns how it went (None = the daemon ran it) + * + * A ReplyTimeout here is NOT a failure: the request reached the daemon and waits for a worker. + */ + RpcFailureKind trySendStop(long timeoutSec = 2); + /** * @brief Disconnect from dragonxd */ @@ -265,6 +282,7 @@ private: // A non-200 reply: RpcError when the body carries a JSON-RPC error, else RpcTransportError classified // from the status and plain-text body. hideBody keeps the body out of the message (secret calls). [[noreturn]] static void throwHttpFailure(long httpCode, const std::string& body, bool hideBody); + bool probeConnection(const std::string& host); // connect()'s getinfo probe, after open() void logConnectFailure(const std::string& msg); // Splits a UnifiedCallback into the (Callback, ErrorCallback) pair used by doRPC. diff --git a/src/util/i18n.cpp b/src/util/i18n.cpp index 8c3df39..6253e48 100644 --- a/src/util/i18n.cpp +++ b/src/util/i18n.cpp @@ -587,6 +587,19 @@ void I18n::loadBuiltinEnglish() strings_["force_quit_confirm_title"] = "Force Quit?"; strings_["force_quit_confirm_msg"] = "This will immediately kill the daemon without a clean shutdown.\nThis may corrupt the blockchain index and require a resync."; strings_["force_quit_yes"] = "Force Quit"; + // Stopping the node on quit: deliver stop, wait, never kill unasked (translations: S10 batch) + strings_["shutdown_leave_running"] = "Close wallet — leave node running"; + strings_["shutdown_leave_running_tip"] = "Closes the wallet now. The node keeps running and finishes shutting down on its own; nothing is lost."; + strings_["shutdown_kill_node"] = "Kill node (unsafe)"; + strings_["shutdown_kill_node_tip"] = "Ends the node at once. Anything it has not saved is lost, and its next start replays the blocks since its last save."; + strings_["shutdown_kill_confirm_title"] = "Kill the node?"; + strings_["shutdown_kill_confirm_msg"] = "The node has shown no CPU or disk activity for a long time and may be stuck.\nKilling it loses everything it has not saved; its next start can take much longer."; + strings_["shutdown_kill_confirm_yes"] = "Kill node"; + strings_["shutdown_stop_asking"] = "Asking the node to stop…"; + strings_["shutdown_stop_queued"] = "The node has the stop request but is busy — it will stop when it can"; + strings_["shutdown_stop_delivered"] = "The node is shutting down and saving its state"; + strings_["shutdown_stop_cpu_suffix"] = " · CPU %d%%"; + strings_["shutdown_stop_idle"] = "No CPU or disk activity for %d min — the node may be stuck"; strings_["reduce_motion"] = "Reduce Motion"; strings_["tt_reduce_motion"] = "Disable animated transitions and balance lerp for accessibility"; strings_["ago"] = "ago"; diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index f83b743..9dd8157 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -80,6 +80,8 @@ #include #include #include +#include +#include #include #include #include @@ -3431,6 +3433,165 @@ void testPosixDestructorLeavesProcessRunning() fsn::remove(script, ec); fsn::remove(pidFile, ec); } + +// executeDaemonStop over a REAL EmbeddedDaemon child: steady clock, real waitpid/signals/CPU sampling. +// No RPC here (stop attempts report NotListening), so on POSIX the group SIGTERM is the stop channel. +class RealStopRuntime final : public dragonx::daemon::DaemonController::StopRuntime { +public: + using DC = dragonx::daemon::DaemonController; + explicit RealStopRuntime(dragonx::daemon::EmbeddedDaemon& d) : d_(d), t0_(std::chrono::steady_clock::now()) {} + double detachAt = -1.0; // choose "Leave running" at this time + double maxCpuPercent = -1.0; + double nowSec() override { return secondsSince(t0_); } + bool processAlive() override { return d_.pollOwnedProcess(); } + dragonx::daemon::StopSendResult trySendStop() override { return dragonx::daemon::StopSendResult::NotListening; } + bool nextCredentials() override { return false; } + bool sendGracefulSignal() override { return d_.signalGracefulStop(); } + double cpuSeconds() override { return d_.ownedResourceUsage().cpuSeconds; } + double ioBytes() override { return -1.0; } + bool rebuildActive() override { return false; } + DC::StopUserChoice userChoice() override + { + return (detachAt >= 0 && nowSec() >= detachAt) ? DC::StopUserChoice::Detach : DC::StopUserChoice::None; + } + void offerForceKill(bool) override {} + void forceKill() override { d_.forceKillOwnedProcess(); } + void setStatus(const DC::StopStatus& s) override { maxCpuPercent = std::max(maxCpuPercent, s.cpuPercent); } + +private: + dragonx::daemon::EmbeddedDaemon& d_; + std::chrono::steady_clock::time_point t0_; +}; + +class RealSleepTask final : public dragonx::daemon::DaemonController::LifecycleTaskContext { +public: + bool cancelled() const override { return false; } + bool shuttingDown() const override { return false; } + void sleepForMs(int ms) override { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); } +}; + +pid_t readPidFile(const std::filesystem::path& pidFile) +{ + for (int i = 0; i < 300; ++i) { + std::ifstream in(pidFile); + long v = 0; + if (in >> v && v > 0) return static_cast(v); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return 0; +} + +// S1: "Leave running" on a node that ignores SIGTERM and keeps a core busy: the stop returns at the next +// tick, nothing kills it, it is still alive afterwards, and its CPU was being measured. +void testPosixStopSessionDetach() +{ + using dragonx::daemon::EmbeddedDaemon; + using DC = dragonx::daemon::DaemonController; + namespace fsn = std::filesystem; + const fsn::path pidFile = fsn::temp_directory_path() / "od_fake_daemon_detach.pid"; + std::error_code ec; + fsn::remove(pidFile, ec); + const auto script = writeFakeDaemonScript("od_fake_daemon_detach.sh", + "trap '' TERM\necho $$ > \"$OD_TEST_PIDFILE\"\nwhile :; do :; done\n"); // pid file = trap is set + setenv("OD_TEST_PIDFILE", pidFile.string().c_str(), 1); + + EmbeddedDaemon d; + d.setSkipPortCheck(true); + EXPECT_TRUE(d.start(script.string())); + const pid_t pid = readPidFile(pidFile); + EXPECT_TRUE(pid > 0); + EXPECT_TRUE(d.beginStop()); + RealStopRuntime rt(d); + rt.detachAt = 2.0; + RealSleepTask task; + DC::StopPolicy p; + p.gracefulSignal = true; + p.userChoices = true; + const auto outcome = DC::executeDaemonStop(p, rt, task); + EXPECT_TRUE(outcome == DC::StopOutcome::Detached); + EXPECT_TRUE(rt.nowSec() < 2.6); + EXPECT_TRUE(rt.maxCpuPercent > 20.0); // the busy loop was measured + d.endStop(/*keepWatching=*/false); + EXPECT_TRUE(pid > 0 && kill(pid, 0) == 0); // left running + EXPECT_FALSE(d.isRunning()); + + if (pid > 0) { + kill(-pid, SIGKILL); + // endStop() left a detached reaper waiting on it: the zombie disappears once it is reaped. + bool reaped = false; + for (int i = 0; i < 200 && !reaped; ++i) { + reaped = (kill(pid, 0) != 0); + if (!reaped) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + EXPECT_TRUE(reaped); + } + unsetenv("OD_TEST_PIDFILE"); + fsn::remove(script, ec); + fsn::remove(pidFile, ec); +} + +// S1/S2 on POSIX: with no RPC answer, the group SIGTERM stops a node that handles it, promptly and +// without counting a crash. +void testPosixStopSessionSigterm() +{ + using dragonx::daemon::EmbeddedDaemon; + using DC = dragonx::daemon::DaemonController; + const auto script = writeFakeDaemonScript("od_fake_daemon_session_term.sh", + "trap 'exit 0' TERM\nwhile :; do sleep 0.1; done\n"); + EmbeddedDaemon d; + d.setSkipPortCheck(true); + EXPECT_TRUE(d.start(script.string())); + EXPECT_TRUE(d.beginStop()); + RealStopRuntime rt(d); + RealSleepTask task; + DC::StopPolicy p; + p.gracefulSignal = true; + p.userChoices = true; + EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == DC::StopOutcome::Exited); + EXPECT_TRUE(rt.nowSec() < 2.0); + d.endStop(/*keepWatching=*/false); + EXPECT_TRUE(d.getState() == EmbeddedDaemon::State::Stopped); + EXPECT_EQ(d.getCrashCount(), 0); + std::error_code ec; std::filesystem::remove(script, ec); +} + +// S1: a bounded wait that gives up keeps watching the node: its later exit is still seen, and booked +// as the clean stop we asked for (not a crash). +void testPosixStopSessionGiveUpKeepsWatching() +{ + using dragonx::daemon::EmbeddedDaemon; + using DC = dragonx::daemon::DaemonController; + namespace fsn = std::filesystem; + const fsn::path readyFile = fsn::temp_directory_path() / "od_fake_daemon_session_slow.ready"; + std::error_code rmEc; + fsn::remove(readyFile, rmEc); + // Signal readiness only once the trap is installed: a SIGTERM before that line would kill it. + const auto script = writeFakeDaemonScript("od_fake_daemon_session_slow.sh", + "trap '' TERM\necho $$ > \"$OD_TEST_PIDFILE\"\nsleep 1.5\nexit 0\n"); + setenv("OD_TEST_PIDFILE", readyFile.string().c_str(), 1); + EmbeddedDaemon d; + d.setSkipPortCheck(true); + EXPECT_TRUE(d.start(script.string())); + EXPECT_TRUE(readPidFile(readyFile) > 0); + EXPECT_TRUE(d.beginStop()); + RealStopRuntime rt(d); + RealSleepTask task; + DC::StopPolicy p; + p.gracefulSignal = true; + p.maxWaitSec = 0.5; + EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == DC::StopOutcome::StillRunning); + d.endStop(/*keepWatching=*/true); + EXPECT_TRUE(d.isRunning()); + for (int i = 0; i < 400 && d.getState() != EmbeddedDaemon::State::Stopped; ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + EXPECT_TRUE(d.getState() == EmbeddedDaemon::State::Stopped); + EXPECT_EQ(d.getCrashCount(), 0); + d.stop(1000); // join the resumed monitor + unsetenv("OD_TEST_PIDFILE"); + std::error_code ec; + fsn::remove(script, ec); + fsn::remove(readyFile, ec); +} #endif // !_WIN32 #ifdef _WIN32 @@ -3481,6 +3642,380 @@ void testWindowsStopNeverKillsForeignDaemon() } #endif // _WIN32 +// --------------------------------------------------------------------------------------------------- +// S2 + S1: delivering "stop" and the never-kill wait (DaemonController::executeDaemonStop), driven by a +// scripted runtime on a virtual clock. +// --------------------------------------------------------------------------------------------------- + +void testStopDeliveryPolicy() +{ + using dragonx::daemon::StopDeliveryPolicy; + using dragonx::daemon::StopSendResult; + using R = StopSendResult; + { // Refused before queueing: not delivered, cheap to retry after 1.5s. + StopDeliveryPolicy p(true); + EXPECT_TRUE(p.shouldSend(0.0)); + EXPECT_TRUE(p.record(R::QueueFull, 0.0) == R::QueueFull); + EXPECT_FALSE(p.shouldSend(1.0)); + EXPECT_TRUE(p.shouldSend(1.5)); + EXPECT_FALSE(p.delivered()); + } + { // Queued: the daemon has it; no resend for 30s (each resend takes a queue slot). + StopDeliveryPolicy p(true); + p.record(R::Queued, 0.0); + EXPECT_TRUE(p.heard()); + EXPECT_FALSE(p.delivered()); + EXPECT_FALSE(p.shouldSend(29.9)); + EXPECT_TRUE(p.shouldSend(30.0)); + // ...and a refusal after that means it unbound its listener to shut down. + EXPECT_TRUE(p.record(R::NotListening, 30.0) == R::ShuttingDown); + EXPECT_TRUE(p.delivered()); + } + { // NotListening before we were ever heard is just "not up / not bound": retry. + StopDeliveryPolicy p(true); + EXPECT_TRUE(p.record(R::NotListening, 0.0) == R::NotListening); + EXPECT_FALSE(p.delivered()); + EXPECT_TRUE(p.shouldSend(1.5)); + } + { // Accepted / ShuttingDown: delivered, never resend. + StopDeliveryPolicy a(true), s(true); + a.record(R::Accepted, 0.0); + s.record(R::ShuttingDown, 0.0); + EXPECT_TRUE(a.delivered() && a.finished() && !a.shouldSend(1000.0)); + EXPECT_TRUE(s.delivered() && s.finished() && !s.shouldSend(1000.0)); + } + { // Auth: terminal for a node we don't own; our own may retry with other credentials. + StopDeliveryPolicy unowned(false), owned(true); + unowned.record(R::Auth, 0.0); + EXPECT_TRUE(unowned.finished() && !unowned.delivered()); + owned.record(R::Auth, 0.0); + EXPECT_FALSE(owned.finished()); + owned.retryNow(0.2); + EXPECT_TRUE(owned.shouldSend(0.2)); + } + using dragonx::daemon::stopSendResultFromKind; + using K = dragonx::rpc::RpcFailureKind; + EXPECT_TRUE(stopSendResultFromKind(K::None) == R::Accepted); + EXPECT_TRUE(stopSendResultFromKind(K::ReplyTimeout) == R::Queued); + EXPECT_TRUE(stopSendResultFromKind(K::QueueFull) == R::QueueFull); + EXPECT_TRUE(stopSendResultFromKind(K::NotListening) == R::NotListening); + EXPECT_TRUE(stopSendResultFromKind(K::ConnectTimeout) == R::NotListening); + EXPECT_TRUE(stopSendResultFromKind(K::ShuttingDown) == R::ShuttingDown); + EXPECT_TRUE(stopSendResultFromKind(K::Unhandled) == R::ShuttingDown); + EXPECT_TRUE(stopSendResultFromKind(K::Auth) == R::Auth); + EXPECT_TRUE(stopSendResultFromKind(K::Aborted) == R::Other); +} + +void testProcStatParse() +{ + using dragonx::daemon::EmbeddedDaemon; + long pgrp = 0; + unsigned long long ticks = 0; + // comm with spaces AND a ')': fields must be read after the LAST ')'. tpgid -1 must parse. + EXPECT_TRUE(EmbeddedDaemon::parseProcStatLine( + "4242 (weird) name) S 1 4240 4240 0 -1 4194560 100 0 0 0 250 50 0 0 20 0 1 0 12345 1000 10", + pgrp, ticks)); + EXPECT_EQ(pgrp, 4240L); + EXPECT_EQ(ticks, 300ULL); + EXPECT_FALSE(EmbeddedDaemon::parseProcStatLine("garbage without parens", pgrp, ticks)); + EXPECT_FALSE(EmbeddedDaemon::parseProcStatLine("1 (x) S 1", pgrp, ticks)); +} + +void testStopProgressTracker() +{ + using Tracker = dragonx::daemon::DaemonController::StopProgressTracker; + { // Flat CPU and I/O: idle time grows from the first sample. + Tracker tr; + for (int s = 0; s <= 300; ++s) tr.add(s, 10.0, 5e6); + EXPECT_NEAR(tr.idleSec(300.0), 300.0, 1e-9); + } + { // 0.1 cores is progress (threshold 0.05). + Tracker tr; + for (int s = 0; s <= 300; ++s) tr.add(s, 0.1 * s, -1.0); + EXPECT_TRUE(tr.idleSec(300.0) < 1.0); + EXPECT_NEAR(tr.cpuPercent(), 10.0, 1e-6); + } + { // A low-CPU flush still writing ~2 MB/min is progress too. + Tracker tr; + for (int s = 0; s <= 300; ++s) tr.add(s, 1.0, 2e6 / 60.0 * s); + EXPECT_TRUE(tr.idleSec(300.0) < 1.0); + } + { // No readings at all: never "idle", so Kill is never offered. + Tracker tr; + for (int s = 0; s <= 3600; s += 10) tr.add(s, -1.0, -1.0); + EXPECT_NEAR(tr.idleSec(3600.0), 0.0, 1e-9); + EXPECT_TRUE(tr.cpuPercent() < 0); + } +} + +void testEvaluateStopWait() +{ + using DC = dragonx::daemon::DaemonController; + using D = DC::StopWaitDecision; + DC::StopPolicy shutdown; // the shutdown screen: owned, unbounded, choices + shutdown.userChoices = true; + DC::StopPolicy background; // a restart/rescan/switch: bounded, no choices + background.maxWaitSec = 120; + background.abandonOnShutdown = true; + auto sample = [] { DC::StopWaitSample s; return s; }; + + auto s = sample(); s.processAlive = false; + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Done); + s = sample(); s.choice = DC::StopUserChoice::Detach; + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Detach); + EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::Wait); // no choices there + s = sample(); s.choice = DC::StopUserChoice::Kill; // Kill without an offer: ignored + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); + s.killOffered = true; + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::ForceKill); + s = sample(); s.delivered = true; s.idleSec = 600; // delivered + idle 10 min: offer + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::OfferForceKill); + s.rebuildActive = true; // ...never during a rebuild + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); + s = sample(); s.delivered = false; s.idleSec = 6000; // never delivered: no offer + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); + s = sample(); s.elapsedSec = 120; + EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::GiveUp); + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); // the shutdown screen has no budget + s = sample(); s.shuttingDown = true; + EXPECT_TRUE(DC::evaluateStopWait(background, s) == D::GiveUp); + EXPECT_TRUE(DC::evaluateStopWait(shutdown, s) == D::Wait); + DC::StopPolicy unowned = shutdown; + unowned.owned = false; + s = sample(); s.delivered = true; s.idleSec = 6000; s.killOffered = true; s.choice = DC::StopUserChoice::Kill; + EXPECT_TRUE(DC::evaluateStopWait(unowned, s) == D::Wait); // never kill what we didn't launch +} + +// A scripted node for executeDaemonStop, on a virtual clock (seconds) that sleeps and sends advance. +class MockStopRuntime final : public dragonx::daemon::DaemonController::StopRuntime { +public: + using DC = dragonx::daemon::DaemonController; + using R = dragonx::daemon::StopSendResult; + + double clock = 0.0; + std::vector sends; // results in order; the last one repeats + double exitAt = -1.0; // the process exits at this time (< 0: not on its own) + double exitAfterDelivery = -1.0; // ...or this long after stop/SIGTERM reached it + bool hasSignal = false; // POSIX-like graceful channel + int credentialsLeft = 0; + std::function cpuAt; // cumulative CPU seconds at t; empty = unknown + std::function choiceAt; + bool rebuild = false; + + std::vector calls; // "stop", "sigterm", "creds", "offer", "withdraw", "kill" + double deliveredAt = -1.0, firstSigtermAt = -1.0, firstOfferAt = -1.0; + double lastStopAt = -1.0; + bool killed = false; + bool hitSafetyLimit = false; + DC::StopStatus lastStatus; + + int count(const std::string& c) const { return (int)std::count(calls.begin(), calls.end(), c); } + + double nowSec() override { return clock; } + bool processAlive() override + { + if (clock > 36000.0) { hitSafetyLimit = true; return false; } // a test bug, not an outcome + if (killed) return false; + if (exitAt >= 0 && clock >= exitAt) return false; + if (exitAfterDelivery >= 0 && deliveredAt >= 0 && clock >= deliveredAt + exitAfterDelivery) return false; + return true; + } + R trySendStop() override + { + calls.push_back("stop"); + lastStopAt = clock; + const size_t n = calls.empty() ? 0 : (size_t)count("stop") - 1; + const R r = sends.empty() ? R::Accepted : sends[std::min(n, sends.size() - 1)]; + clock += (r == R::Queued) ? 2.0 : 0.01; // a reply timeout costs the client timeout + if ((r == R::Accepted || r == R::ShuttingDown) && deliveredAt < 0) deliveredAt = clock; + return r; + } + bool nextCredentials() override + { + calls.push_back("creds"); + return credentialsLeft-- > 0; + } + bool sendGracefulSignal() override + { + if (!hasSignal) return false; + calls.push_back("sigterm"); + if (firstSigtermAt < 0) firstSigtermAt = clock; + if (deliveredAt < 0) deliveredAt = clock; + return true; + } + double cpuSeconds() override { return cpuAt ? cpuAt(clock) : -1.0; } + double ioBytes() override { return -1.0; } + bool rebuildActive() override { return rebuild; } + DC::StopUserChoice userChoice() override { return choiceAt ? choiceAt(clock) : DC::StopUserChoice::None; } + void offerForceKill(bool offered) override + { + calls.push_back(offered ? "offer" : "withdraw"); + if (offered && firstOfferAt < 0) firstOfferAt = clock; + } + void forceKill() override + { + calls.push_back("kill"); + killed = true; + } + void setStatus(const DC::StopStatus& s) override { lastStatus = s; } +}; + +class MockStopTask final : public dragonx::daemon::DaemonController::LifecycleTaskContext { +public: + explicit MockStopTask(MockStopRuntime& rt) : rt_(rt) {} + double shutdownAt = -1.0; + bool cancelled() const override { return false; } + bool shuttingDown() const override { return shutdownAt >= 0 && rt_.clock >= shutdownAt; } + void sleepForMs(int ms) override { rt_.clock += ms / 1000.0; } + +private: + MockStopRuntime& rt_; +}; + +dragonx::daemon::DaemonController::StopPolicy shutdownStopPolicy(bool posix) +{ + dragonx::daemon::DaemonController::StopPolicy p; + p.owned = true; + p.gracefulSignal = posix; + p.userChoices = true; + return p; +} + +void testDaemonStopExecution() +{ + using DC = dragonx::daemon::DaemonController; + using R = dragonx::daemon::StopSendResult; + using O = DC::StopOutcome; + + { // Already exited: not a single RPC call. + MockStopRuntime rt; MockStopTask task(rt); + rt.exitAt = 0.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_TRUE(rt.calls.empty()); + } + { // The 2026-09-30 12:21:44 -> 12:22:04 regression: the queue is full for a while. Retry until it + // lands (Windows-like, no signal channel), then wait for the exit. Never kill. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::QueueFull, R::QueueFull, R::QueueFull, R::QueueFull, R::QueueFull, R::Accepted}; + rt.exitAfterDelivery = 3.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_EQ(rt.count("stop"), 6); + EXPECT_EQ(rt.count("kill"), 0); + EXPECT_TRUE(rt.deliveredAt >= 5 * 1.5); // retries were spaced, not a tight loop + EXPECT_FALSE(rt.hitSafetyLimit); + } + { // Never automatic: queue full for an hour, CPU busy (a rebuild with no log marker). No kill, no + // offer; the node exits on its own at 60 min. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::QueueFull}; + rt.exitAt = 3600.0; + rt.cpuAt = [](double t) { return 6.0 * t; }; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_EQ(rt.count("kill"), 0); + EXPECT_EQ(rt.count("offer"), 0); + EXPECT_TRUE(rt.count("stop") > 1000); // still trying every 1.5s + EXPECT_FALSE(rt.hitSafetyLimit); + } + { // POSIX: the first attempt is refused, so the group SIGTERM goes out at once (<= 2s), and no + // more RPC attempts are needed. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::QueueFull}; + rt.hasSignal = true; + rt.exitAfterDelivery = 2.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited); + EXPECT_TRUE(rt.firstSigtermAt >= 0 && rt.firstSigtermAt <= 2.0); + EXPECT_EQ(rt.count("stop"), 1); + EXPECT_EQ(rt.count("kill"), 0); + } + { // Queued (reply timeout) is not a failure: no SIGTERM, no resend before 30s. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::Queued}; + rt.hasSignal = true; + rt.exitAt = 25.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited); + EXPECT_EQ(rt.count("stop"), 1); + EXPECT_EQ(rt.count("sigterm"), 0); + } + { // Delivered, then flat CPU: Kill is OFFERED after ~10 min, and only an explicit Kill kills (once). + MockStopRuntime rt; MockStopTask task(rt); + rt.cpuAt = [](double) { return 42.0; }; + rt.choiceAt = [](double t) { return t >= 900.0 ? DC::StopUserChoice::Kill : DC::StopUserChoice::None; }; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Killed); + EXPECT_TRUE(rt.firstOfferAt >= 600.0 && rt.firstOfferAt < 602.0); + EXPECT_EQ(rt.count("kill"), 1); + EXPECT_TRUE(rt.clock >= 900.0); // waited for the user, didn't act on the offer itself + } + { // ...with no answer to the offer, it keeps waiting (and never kills). + MockStopRuntime rt; MockStopTask task(rt); + rt.cpuAt = [](double) { return 42.0; }; + rt.exitAt = 3600.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_EQ(rt.count("offer"), 1); + EXPECT_EQ(rt.count("kill"), 0); + } + { // A rebuild in progress suppresses the offer even with flat readings. + MockStopRuntime rt; MockStopTask task(rt); + rt.cpuAt = [](double) { return 42.0; }; + rt.rebuild = true; + rt.exitAt = 1800.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_EQ(rt.count("offer"), 0); + } + { // "Leave running": returns at once, no kill, no further stop attempts. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::QueueFull}; + rt.choiceAt = [](double t) { return t >= 5.0 ? DC::StopUserChoice::Detach : DC::StopUserChoice::None; }; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Detached); + EXPECT_EQ(rt.count("kill"), 0); + EXPECT_TRUE(rt.lastStopAt < 5.0); + EXPECT_TRUE(rt.clock < 5.5); + } + { // A node we didn't launch is never killed, even with Kill chosen; a bounded wait then gives up. + MockStopRuntime rt; MockStopTask task(rt); + DC::StopPolicy p = shutdownStopPolicy(false); + p.owned = false; + p.maxWaitSec = 120.0; + rt.choiceAt = [](double) { return DC::StopUserChoice::Kill; }; + EXPECT_TRUE(DC::executeDaemonStop(p, rt, task) == O::StillRunning); + EXPECT_EQ(rt.count("kill"), 0); + EXPECT_EQ(rt.count("offer"), 0); + EXPECT_TRUE(rt.clock >= 120.0 && rt.clock < 121.0); + } + { // Background callers (restart, rescan, switch): bounded and never kill; they also let go as soon + // as the app starts shutting down, so the shutdown can take over. + DC::StopPolicy bg; + bg.maxWaitSec = 120.0; + bg.abandonOnShutdown = true; + MockStopRuntime a; MockStopTask ta(a); + a.cpuAt = [](double) { return 1.0; }; + EXPECT_TRUE(DC::executeDaemonStop(bg, a, ta) == O::StillRunning); + EXPECT_EQ(a.count("kill"), 0); + EXPECT_TRUE(a.clock >= 120.0 && a.clock < 121.0); + MockStopRuntime b; MockStopTask tb(b); + tb.shutdownAt = 10.0; + EXPECT_TRUE(DC::executeDaemonStop(bg, b, tb) == O::StillRunning); + EXPECT_TRUE(b.clock >= 10.0 && b.clock < 11.0); + } + { // Our own node refusing our credentials: try the next source, then deliver. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::Auth, R::Auth, R::Accepted}; + rt.credentialsLeft = 2; + rt.exitAfterDelivery = 1.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(false), rt, task) == O::Exited); + EXPECT_EQ(rt.count("creds"), 2); + EXPECT_EQ(rt.count("stop"), 3); + } + { // ...out of credential sources on POSIX: the graceful signal still stops it. + MockStopRuntime rt; MockStopTask task(rt); + rt.sends = {R::Auth}; + rt.hasSignal = true; + rt.exitAfterDelivery = 1.0; + EXPECT_TRUE(DC::executeDaemonStop(shutdownStopPolicy(true), rt, task) == O::Exited); + EXPECT_EQ(rt.count("sigterm"), 1); + EXPECT_EQ(rt.count("kill"), 0); + } +} + // S3: on Linux a node in its final shutdown phase reads comm "hush-shutoff"; its executable still says // dragonxd, so it must still count as running (it holds the datadir lock until it exits). void testDaemonProcessIdentity() @@ -8064,6 +8599,11 @@ int main() testDaemonShutdownPolicy(); testDatadirLockGate(); testDaemonProcessIdentity(); + testStopDeliveryPolicy(); + testProcStatParse(); + testStopProgressTracker(); + testEvaluateStopWait(); + testDaemonStopExecution(); #ifdef _WIN32 testWindowsStopNeverKillsForeignDaemon(); #endif @@ -8074,6 +8614,9 @@ int main() testPosixRequestedStopIsNotACrash(); testPosixStopSigtermIsGraceful(); testPosixDestructorLeavesProcessRunning(); + testPosixStopSessionDetach(); + testPosixStopSessionSigterm(); + testPosixStopSessionGiveUpKeepsWatching(); #endif testPlatformEnsureDirectory(); testVerifySaplingParams();