DanSandClaude Opus 5.5
5993488176
Closing the wallet sent "stop" at most twice, gated on a getinfo probe that needs the daemon's cs_main, then killed the node after 20 s (Windows) or 30 s (POSIX) whether or not stop had arrived. A node busy with an hour-long witness rebuild therefore never got stop, was killed mid- rebuild, and replayed everything since its last flush on every start. Delivery (src/daemon/stop_delivery.h, RPCClient::open/trySendStop): - stop goes over a dedicated client with no probe, so it never waits on rpc_'s lock or a cs_main-bound getinfo - every attempt is classified, and StopDeliveryPolicy decides the next: Accepted / 503 = delivered; Queued (reply timeout) is resent at most every 30 s; QueueFull / not listening are retried every 1.5 s; a refusal after we were heard means the node is shutting down; a 401 on our own node tries the next credential source - POSIX: a group SIGTERM goes out as soon as an attempt fails, which bypasses the HTTP queue entirely Waiting (DaemonController::executeDaemonStop, pure, virtual-clock tested): - never kills on its own. Quitting waits as long as the node needs; the shutdown screen offers "Close wallet — leave node running" after 3 s, and "Kill node (unsafe)" (confirmed) only once stop was delivered and the node showed no CPU/I-O progress for ~10 min outside a rebuild - other callers (restart, rescan, reinstall, switch, ...) wait up to 120 s, let go once the app starts quitting, and keep watching a node that is still running; wallet switch refuses to go ahead then - EmbeddedDaemon gains stop-session primitives (beginStop, pollOwnedProcess, signalGracefulStop, forceKillOwnedProcess, ownedResourceUsage over the Linux process group / Windows handle, endStop) and only one stop session runs at a time - the Force Quit button and its stall heuristic are gone; "Leave running" aborts a pending stop attempt and closes at once Verified against the fake daemon with the real GUI: closing mid-rebuild now delivers stop (or SIGTERM 2 ms after a queue-full refusal) and the node exits cleanly after its rebuild, where it was SIGKILLed before. The shutdown screen, its "Leave running" button and the idle Kill offer (after ~11 min) were checked by screenshot. New strings are English- only for now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
454 lines
19 KiB
C++
454 lines
19 KiB
C++
#pragma once
|
|
|
|
#include "embedded_daemon.h"
|
|
#include "stop_delivery.h"
|
|
|
|
#include <algorithm>
|
|
#include <cstddef>
|
|
#include <deque>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace dragonx {
|
|
namespace config { class Settings; }
|
|
namespace daemon {
|
|
|
|
class DaemonController {
|
|
public:
|
|
using State = EmbeddedDaemon::State;
|
|
using StateCallback = EmbeddedDaemon::StateCallback;
|
|
|
|
enum class ShutdownAction {
|
|
DisconnectOnly,
|
|
StopDaemon
|
|
};
|
|
|
|
struct ShutdownDecision {
|
|
ShutdownAction action = ShutdownAction::DisconnectOnly;
|
|
const char* logReason = "no embedded daemon";
|
|
const char* status = "Disconnecting...";
|
|
};
|
|
|
|
enum class LifecycleOperation {
|
|
ManualRestart,
|
|
Rescan,
|
|
RepairWallet, // restart with -zapwallettxes=2 (wipe & rebuild wallet tx records)
|
|
DeleteBlockchainData,
|
|
BootstrapStop
|
|
};
|
|
|
|
struct LifecycleDecision {
|
|
LifecycleOperation operation = LifecycleOperation::ManualRestart;
|
|
bool allowed = false;
|
|
bool wasRunning = false;
|
|
const char* taskName = "";
|
|
const char* status = "";
|
|
const char* warning = "";
|
|
bool resetCrashCount = false;
|
|
bool setRescanOnNextStart = false;
|
|
bool disconnectRpc = false;
|
|
int restartDelayMs = 0;
|
|
bool setZapOnNextStart = false;
|
|
};
|
|
|
|
class LifecycleTaskContext {
|
|
public:
|
|
virtual ~LifecycleTaskContext() = default;
|
|
virtual bool cancelled() const = 0;
|
|
virtual bool shuttingDown() const = 0;
|
|
virtual void sleepForMs(int milliseconds) = 0;
|
|
};
|
|
|
|
class LifecycleRuntime {
|
|
public:
|
|
virtual ~LifecycleRuntime() = default;
|
|
virtual void stopDaemonWithPolicy() = 0;
|
|
virtual bool startDaemon() = 0;
|
|
virtual int deleteBlockchainData() = 0;
|
|
virtual void resetOutputOffset() = 0;
|
|
virtual void requestRpcStopAndDisconnect(const char* context, const char* reason) = 0;
|
|
};
|
|
|
|
struct LifecycleExecutionResult {
|
|
bool completed = false;
|
|
bool cancelled = false;
|
|
bool stopped = false;
|
|
bool started = false;
|
|
int deletedItems = 0;
|
|
};
|
|
|
|
DaemonController();
|
|
~DaemonController();
|
|
|
|
DaemonController(const DaemonController&) = delete;
|
|
DaemonController& operator=(const DaemonController&) = delete;
|
|
|
|
EmbeddedDaemon* daemon() { return daemon_.get(); }
|
|
const EmbeddedDaemon* daemon() const { return daemon_.get(); }
|
|
|
|
void setStateCallback(StateCallback callback);
|
|
void syncSettings(const config::Settings* settings);
|
|
|
|
bool start(const config::Settings* settings);
|
|
void stop(int waitMs);
|
|
void markStopRequested();
|
|
|
|
bool isRunning() const;
|
|
bool externalDaemonDetected() const;
|
|
void clearExternalDaemonDetected();
|
|
State state() const;
|
|
std::string lastError() const; // by value: EmbeddedDaemon::getLastError() returns a locked copy (M-04)
|
|
int crashCount() const;
|
|
int lastBlockHeight() const;
|
|
double memoryUsageMB() const;
|
|
std::vector<std::string> recentLines(std::size_t count) const;
|
|
std::string outputSince(std::size_t& offset) const;
|
|
|
|
void resetCrashCount();
|
|
void setRescanOnNextStart(bool enabled);
|
|
bool rescanOnNextStart() const;
|
|
void setZapOnNextStart(bool enabled);
|
|
bool zapOnNextStart() const;
|
|
void setSalvageOnNextStart(bool enabled);
|
|
void setReindexOnNextStart(bool enabled); // -reindex: rebuild the block DB from raw blocks on next start
|
|
|
|
static ShutdownDecision evaluateShutdownPolicy(bool hasDaemon,
|
|
bool externalDaemonDetected,
|
|
bool keepDaemonRunning,
|
|
bool stopExternalDaemon) {
|
|
if (!hasDaemon) {
|
|
return {};
|
|
}
|
|
if (keepDaemonRunning) {
|
|
return {ShutdownAction::DisconnectOnly,
|
|
"keep_daemon_running enabled",
|
|
"Disconnecting (daemon stays running)..."};
|
|
}
|
|
if (externalDaemonDetected && !stopExternalDaemon) {
|
|
return {ShutdownAction::DisconnectOnly,
|
|
"external daemon (not ours to stop)",
|
|
"Disconnecting (daemon stays running)..."};
|
|
}
|
|
return {ShutdownAction::StopDaemon,
|
|
"stopping managed daemon",
|
|
"Sending stop command to daemon..."};
|
|
}
|
|
static LifecycleDecision evaluateLifecycleOperation(LifecycleOperation operation,
|
|
bool usingEmbeddedDaemon,
|
|
bool hasDaemon,
|
|
bool daemonRunning,
|
|
bool restartInProgress = false) {
|
|
switch (operation) {
|
|
case LifecycleOperation::ManualRestart:
|
|
if (!usingEmbeddedDaemon || restartInProgress) return {};
|
|
return {operation, true, daemonRunning, "daemon-restart", "Restarting daemon...", "",
|
|
true, false, true, 500};
|
|
case LifecycleOperation::Rescan:
|
|
if (!usingEmbeddedDaemon || !hasDaemon) {
|
|
return {operation, false, daemonRunning, "", "",
|
|
"Rescan requires embedded daemon. Restart your daemon with -rescan manually."};
|
|
}
|
|
return {operation, true, daemonRunning, "rescan-blockchain", "Starting rescan...", "",
|
|
false, true, false, 3000};
|
|
case LifecycleOperation::RepairWallet:
|
|
if (!usingEmbeddedDaemon || !hasDaemon) {
|
|
return {operation, false, daemonRunning, "", "",
|
|
"Wallet repair requires embedded daemon. Restart your daemon with -zapwallettxes=2 manually."};
|
|
}
|
|
return {operation, true, daemonRunning, "repair-wallet", "Repairing wallet...", "",
|
|
false, false, false, 3000, true};
|
|
case LifecycleOperation::DeleteBlockchainData:
|
|
if (!usingEmbeddedDaemon || !hasDaemon) {
|
|
return {operation, false, daemonRunning, "", "",
|
|
"Delete blockchain requires embedded daemon. Stop your daemon manually and delete the data directory."};
|
|
}
|
|
return {operation, true, daemonRunning, "delete-blockchain-data", "Deleting blockchain data...", "",
|
|
false, false, false, 3000};
|
|
case LifecycleOperation::BootstrapStop:
|
|
return {operation, true, daemonRunning, "bootstrap-stop-daemon", "Stopping daemon for bootstrap...", "",
|
|
false, false, true, 0};
|
|
}
|
|
return {};
|
|
}
|
|
void prepareLifecycleOperation(const LifecycleDecision& decision,
|
|
const config::Settings* settings = nullptr);
|
|
static inline LifecycleExecutionResult executeLifecycleOperation(const LifecycleDecision& decision,
|
|
LifecycleRuntime& runtime,
|
|
LifecycleTaskContext& task)
|
|
{
|
|
LifecycleExecutionResult result;
|
|
if (!decision.allowed) return result;
|
|
|
|
auto waitForDelay = [&]() {
|
|
int waitTicks = std::max(0, decision.restartDelayMs / 100);
|
|
for (int i = 0; i < waitTicks && !task.cancelled() && !task.shuttingDown(); ++i) {
|
|
task.sleepForMs(100);
|
|
}
|
|
};
|
|
|
|
auto cancelled = [&]() {
|
|
result.cancelled = task.cancelled() || task.shuttingDown();
|
|
return result.cancelled;
|
|
};
|
|
|
|
switch (decision.operation) {
|
|
case LifecycleOperation::BootstrapStop:
|
|
if (decision.wasRunning) {
|
|
runtime.requestRpcStopAndDisconnect("Bootstrap daemon stop", "Bootstrap");
|
|
result.stopped = true;
|
|
}
|
|
result.completed = true;
|
|
return result;
|
|
case LifecycleOperation::ManualRestart:
|
|
if (decision.wasRunning) {
|
|
runtime.stopDaemonWithPolicy();
|
|
result.stopped = true;
|
|
}
|
|
break;
|
|
case LifecycleOperation::Rescan:
|
|
case LifecycleOperation::RepairWallet:
|
|
case LifecycleOperation::DeleteBlockchainData:
|
|
runtime.stopDaemonWithPolicy();
|
|
result.stopped = true;
|
|
break;
|
|
}
|
|
|
|
if (cancelled()) return result;
|
|
waitForDelay();
|
|
if (cancelled()) return result;
|
|
|
|
if (decision.operation == LifecycleOperation::DeleteBlockchainData) {
|
|
result.deletedItems = runtime.deleteBlockchainData();
|
|
if (cancelled()) return result;
|
|
}
|
|
|
|
if (decision.operation == LifecycleOperation::Rescan ||
|
|
decision.operation == LifecycleOperation::RepairWallet ||
|
|
decision.operation == LifecycleOperation::DeleteBlockchainData) {
|
|
runtime.resetOutputOffset();
|
|
}
|
|
|
|
result.started = runtime.startDaemon();
|
|
result.completed = !cancelled();
|
|
return result;
|
|
}
|
|
ShutdownDecision shutdownDecision(bool keepDaemonRunning,
|
|
bool stopExternalDaemon) const;
|
|
|
|
// ------------------------------------------------------------------------------------------------
|
|
// Stopping the node: deliver "stop" until it lands, then wait for the process to exit. Never kill it
|
|
// on our own: a node that is busy (a witness rebuild can hold it for an hour) or flushing would lose
|
|
// everything since its last save and replay it on the next start. Only an explicit user "Kill",
|
|
// offered once the node has gone idle, ends it by force; "Leave running" lets the wallet close while
|
|
// the node finishes on its own.
|
|
// ------------------------------------------------------------------------------------------------
|
|
|
|
using StopUserChoice = daemon::StopUserChoice;
|
|
using StopOutcome = daemon::StopOutcome;
|
|
|
|
struct StopPolicy {
|
|
bool owned = true; // we launched it: watch the process, signal it, offer Kill
|
|
bool gracefulSignal = false; // POSIX: a group SIGTERM is a stop channel (dragonxd treats it as stop)
|
|
double maxWaitSec = 0.0; // stop waiting (never kill) after this long; 0 = no limit
|
|
bool userChoices = false; // the shutdown screen offers "Leave running" / "Kill"
|
|
bool abandonOnShutdown = false; // stop waiting once the app starts shutting down (background callers)
|
|
double idleOfferSec = 600.0; // delivered + no CPU/I-O progress for this long: offer Kill
|
|
};
|
|
|
|
// What the stop loop reports for display.
|
|
struct StopStatus {
|
|
bool delivered = false; // stop accepted (or the node is shutting down, or was signalled)
|
|
bool heard = false; // the node has the request (possibly still queued)
|
|
bool signalled = false; // a graceful signal (POSIX SIGTERM) went out
|
|
double elapsedSec = 0.0;
|
|
double cpuPercent = -1.0; // < 0: unknown
|
|
double idleSec = 0.0;
|
|
bool killOffered = false;
|
|
bool rebuildActive = false;
|
|
StopSendResult lastSend = StopSendResult::Other;
|
|
int attempts = 0;
|
|
};
|
|
|
|
class StopRuntime {
|
|
public:
|
|
virtual ~StopRuntime() = default;
|
|
virtual double nowSec() = 0;
|
|
virtual bool processAlive() = 0;
|
|
virtual StopSendResult trySendStop() = 0;
|
|
virtual bool nextCredentials() = 0; // switch to the next stop credentials; false when none left
|
|
virtual bool sendGracefulSignal() = 0; // POSIX group SIGTERM; false where there is none (Windows)
|
|
virtual double cpuSeconds() = 0; // cumulative CPU of the node; < 0 if unknown
|
|
virtual double ioBytes() = 0; // cumulative I/O bytes of the node; < 0 if unknown
|
|
virtual bool rebuildActive() = 0; // a witness rebuild is running: never offer Kill
|
|
virtual StopUserChoice userChoice() = 0;
|
|
virtual void offerForceKill(bool offered) = 0;
|
|
virtual void forceKill() = 0;
|
|
virtual void setStatus(const StopStatus& status) = 0;
|
|
};
|
|
|
|
/**
|
|
* "Making progress" = >= 0.05 cores of CPU or >= 1 MB of I/O over the last >= 60s. A flush that
|
|
* fsyncs a big wallet is mostly I/O at low CPU; a rebuild is mostly CPU. Idle time counts from the
|
|
* first sample. With no readings at all (unknown), it never reports idle, so Kill is never offered.
|
|
*/
|
|
class StopProgressTracker {
|
|
public:
|
|
static constexpr double kWindowSec = 60.0;
|
|
static constexpr double kMinCores = 0.05;
|
|
static constexpr double kMinIoBytes = 1024.0 * 1024.0;
|
|
|
|
void add(double t, double cpuSec, double ioBytes)
|
|
{
|
|
if (cpuSec >= 0 || ioBytes >= 0) measurable_ = true;
|
|
if (samples_.empty()) lastProgressAt_ = t;
|
|
samples_.push_back({t, cpuSec, ioBytes});
|
|
// Keep the newest sample that is at least a full window old as the comparison base.
|
|
while (samples_.size() > 2 && samples_[1].t <= t - kWindowSec) samples_.pop_front();
|
|
const Sample& base = samples_.front();
|
|
const double span = t - base.t;
|
|
if (span >= kWindowSec) {
|
|
const bool cpuMoved = cpuSec >= 0 && base.cpu >= 0 && (cpuSec - base.cpu) / span >= kMinCores;
|
|
const bool ioMoved = ioBytes >= 0 && base.io >= 0 && (ioBytes - base.io) >= kMinIoBytes;
|
|
if (cpuMoved || ioMoved) lastProgressAt_ = t;
|
|
}
|
|
}
|
|
double idleSec(double t) const { return measurable_ ? std::max(0.0, t - lastProgressAt_) : 0.0; }
|
|
// CPU over the most recent sample interval (100 = one core), for display; < 0 unknown.
|
|
double cpuPercent() const
|
|
{
|
|
if (samples_.size() < 2) return -1.0;
|
|
const Sample& a = samples_[samples_.size() - 2];
|
|
const Sample& b = samples_.back();
|
|
if (a.cpu < 0 || b.cpu < 0 || b.t <= a.t) return -1.0;
|
|
return std::max(0.0, (b.cpu - a.cpu) / (b.t - a.t) * 100.0);
|
|
}
|
|
|
|
private:
|
|
struct Sample { double t, cpu, io; };
|
|
std::deque<Sample> samples_;
|
|
double lastProgressAt_ = 0.0;
|
|
bool measurable_ = false;
|
|
};
|
|
|
|
enum class StopWaitDecision { Done, Wait, GiveUp, Detach, OfferForceKill, ForceKill };
|
|
|
|
struct StopWaitSample {
|
|
bool processAlive = true;
|
|
bool delivered = false;
|
|
double elapsedSec = 0.0;
|
|
double idleSec = 0.0;
|
|
bool rebuildActive = false;
|
|
StopUserChoice choice = StopUserChoice::None;
|
|
bool killOffered = false; // the Kill offer is on screen right now
|
|
bool shuttingDown = false; // the app began shutting down / the task was cancelled
|
|
};
|
|
|
|
static inline StopWaitDecision evaluateStopWait(const StopPolicy& p, const StopWaitSample& s)
|
|
{
|
|
if (!s.processAlive) return StopWaitDecision::Done;
|
|
if (p.userChoices && s.choice == StopUserChoice::Detach) return StopWaitDecision::Detach;
|
|
if (p.userChoices && p.owned && s.choice == StopUserChoice::Kill && s.killOffered)
|
|
return StopWaitDecision::ForceKill;
|
|
if (p.abandonOnShutdown && s.shuttingDown) return StopWaitDecision::GiveUp;
|
|
if (p.maxWaitSec > 0 && s.elapsedSec >= p.maxWaitSec) return StopWaitDecision::GiveUp;
|
|
if (p.userChoices && p.owned && s.delivered && !s.rebuildActive && s.idleSec >= p.idleOfferSec)
|
|
return StopWaitDecision::OfferForceKill;
|
|
return StopWaitDecision::Wait;
|
|
}
|
|
|
|
static constexpr int kStopTickMs = 250;
|
|
|
|
static inline StopOutcome executeDaemonStop(const StopPolicy& policy, StopRuntime& rt,
|
|
LifecycleTaskContext& task)
|
|
{
|
|
if (!rt.processAlive()) return StopOutcome::Exited; // already gone: not a single RPC call
|
|
|
|
const double t0 = rt.nowSec();
|
|
StopDeliveryPolicy delivery(policy.owned);
|
|
StopProgressTracker progress;
|
|
bool signalled = false;
|
|
bool offered = false;
|
|
auto withdrawOffer = [&] {
|
|
if (offered) { offered = false; rt.offerForceKill(false); }
|
|
};
|
|
|
|
for (;;) {
|
|
double now = rt.nowSec() - t0;
|
|
if (delivery.shouldSend(now)) {
|
|
const StopSendResult r = delivery.record(rt.trySendStop(), now);
|
|
if (r == StopSendResult::Auth && policy.owned) {
|
|
// Our own node refusing our credentials means they're stale (e.g. a regenerated
|
|
// .cookie): try the next source; out of sources, the graceful signal below remains.
|
|
if (rt.nextCredentials()) delivery.retryNow(rt.nowSec() - t0);
|
|
else delivery.giveUp();
|
|
}
|
|
// POSIX: a group SIGTERM bypasses the RPC queue altogether (dragonxd's handler sets the same
|
|
// flag as stop). Send it as soon as an attempt shows stop didn't get through.
|
|
if (policy.gracefulSignal && !signalled && !delivery.delivered() &&
|
|
r != StopSendResult::Queued && rt.sendGracefulSignal()) {
|
|
signalled = true;
|
|
delivery.markDelivered();
|
|
}
|
|
}
|
|
if (!rt.processAlive()) {
|
|
withdrawOffer();
|
|
return StopOutcome::Exited;
|
|
}
|
|
|
|
now = rt.nowSec() - t0;
|
|
progress.add(now, rt.cpuSeconds(), rt.ioBytes());
|
|
StopWaitSample s;
|
|
s.delivered = delivery.delivered();
|
|
s.elapsedSec = now;
|
|
s.idleSec = progress.idleSec(now);
|
|
s.rebuildActive = rt.rebuildActive();
|
|
s.choice = rt.userChoice();
|
|
s.killOffered = offered;
|
|
s.shuttingDown = task.cancelled() || task.shuttingDown();
|
|
|
|
switch (evaluateStopWait(policy, s)) {
|
|
case StopWaitDecision::Done:
|
|
withdrawOffer();
|
|
return StopOutcome::Exited;
|
|
case StopWaitDecision::Detach:
|
|
withdrawOffer();
|
|
return StopOutcome::Detached;
|
|
case StopWaitDecision::ForceKill:
|
|
withdrawOffer();
|
|
rt.forceKill();
|
|
return StopOutcome::Killed;
|
|
case StopWaitDecision::GiveUp:
|
|
withdrawOffer();
|
|
return StopOutcome::StillRunning;
|
|
case StopWaitDecision::OfferForceKill:
|
|
if (!offered) { offered = true; rt.offerForceKill(true); }
|
|
break;
|
|
case StopWaitDecision::Wait:
|
|
withdrawOffer(); // progress resumed (or a rebuild started): the node isn't stuck after all
|
|
break;
|
|
}
|
|
|
|
StopStatus st;
|
|
st.delivered = delivery.delivered();
|
|
st.heard = delivery.heard();
|
|
st.signalled = signalled;
|
|
st.elapsedSec = now;
|
|
st.cpuPercent = progress.cpuPercent();
|
|
st.idleSec = s.idleSec;
|
|
st.killOffered = offered;
|
|
st.rebuildActive = s.rebuildActive;
|
|
st.lastSend = delivery.last();
|
|
st.attempts = delivery.attempts();
|
|
rt.setStatus(st);
|
|
task.sleepForMs(kStopTickMs);
|
|
}
|
|
}
|
|
|
|
private:
|
|
std::unique_ptr<EmbeddedDaemon> daemon_;
|
|
};
|
|
|
|
} // namespace daemon
|
|
} // namespace dragonx
|