feat(daemon): recognise our detached node, tail POSIX debug.log, more RPC threads
S12 - our node across sessions: - The node the wallet launches is recorded in <datadir>/obsidiandragon- dragonxd.pid (pid + process start time + exe). A later session that finds it still running (after "Leave running" / keep_daemon_running) stops it gracefully on quit and waits for that exact process (no time limit, "Leave running" offered); never signals or kills it. A reused pid or a reboot never matches; stale records are deleted. - stop_external_daemon is now honoured when this session only connected (no controller); it was ignored. S13 - the node's own log on every platform: - No -printtoconsole on Linux/macOS: the node writes its timestamped debug.log (which outlives the GUI) and we tail it next to the stderr pipe. Tail path from the datadir or the isolated override (Windows had %APPDATA% hard-coded, wrong for migrate-to-seed starts). - After the node shrinks debug.log at startup, resume at this run's banner instead of re-reading the old run's tail. - A node we didn't launch (left running, or found running) feeds the busy tracker from its debug.log, tied to the live process (our record, or POSIX hushd.pid) and only from lines it wrote. A GUI attaching to a node mid-rebuild now says "Node busy: rebuilding note witnesses" and asks before stopping it. - A start that only finds a running node no longer resets the tracker; a found busy node shows the busy overlay at once. S14: launch our node with -rpcthreads=16 unless DRAGONX.conf sets it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+153
-37
@@ -944,6 +944,7 @@ void App::update()
|
||||
// The node is too busy to answer: send it nothing that needs cs_main (it would only queue behind the
|
||||
// rest and hold one of its few RPC slots) until the canary gets through. pumpRpcBusyCanary() runs it.
|
||||
pumpRpcBusyCanary();
|
||||
pumpExternalDaemonLog();
|
||||
const bool rpcBusy = rpc_busy_gate_->busy();
|
||||
|
||||
// Keep the "node starting / warming up / busy" overlay current between connect attempts and polls,
|
||||
@@ -5430,8 +5431,11 @@ bool App::startEmbeddedDaemon()
|
||||
});
|
||||
}
|
||||
|
||||
resetDaemonBusy(); // a new process: whatever the last one was doing is over
|
||||
return daemon_controller_->start(settings_.get());
|
||||
// Only a process we actually launched starts a new story: a start that finds a node already running (and
|
||||
// launches nothing) must not wipe what the tracker knows about that node from its debug.log.
|
||||
const bool launched = daemon_controller_->start(settings_.get());
|
||||
if (launched) resetDaemonBusy(); // a new process: whatever the last one was doing is over
|
||||
return launched;
|
||||
}
|
||||
|
||||
// Drives one stop session for DaemonController::executeDaemonStop: a dedicated RPC client for "stop" (not
|
||||
@@ -5441,8 +5445,10 @@ class AppStopRuntime final : public daemon::DaemonController::StopRuntime {
|
||||
public:
|
||||
using DC = daemon::DaemonController;
|
||||
|
||||
AppStopRuntime(App& app, daemon::EmbeddedDaemon* owned, std::vector<rpc::ConnectionConfig> credentials)
|
||||
: app_(app), owned_(owned), creds_(std::move(credentials)), t0_(std::chrono::steady_clock::now())
|
||||
AppStopRuntime(App& app, daemon::EmbeddedDaemon* owned, std::vector<rpc::ConnectionConfig> credentials,
|
||||
daemon::EmbeddedDaemon::OwnershipRecord recorded = {})
|
||||
: app_(app), owned_(owned), creds_(std::move(credentials)), recorded_(std::move(recorded)),
|
||||
t0_(std::chrono::steady_clock::now())
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(app_.stop_client_mutex_);
|
||||
app_.active_stop_client_ = &client_;
|
||||
@@ -5460,6 +5466,9 @@ public:
|
||||
bool processAlive() override
|
||||
{
|
||||
if (owned_) return owned_->pollOwnedProcess();
|
||||
// The node an earlier session of ours left running: that exact process (pid + start time).
|
||||
if (recorded_.pid > 0)
|
||||
return daemon::EmbeddedDaemon::processStartToken(recorded_.pid) == recorded_.startToken;
|
||||
// Not ours: we can only watch for it to go (port and process), never wait on or signal it.
|
||||
return daemon::EmbeddedDaemon::isRpcPortInUse() || daemon::EmbeddedDaemon::isDaemonProcessRunning();
|
||||
}
|
||||
@@ -5536,6 +5545,7 @@ private:
|
||||
App& app_;
|
||||
daemon::EmbeddedDaemon* owned_ = nullptr;
|
||||
std::vector<rpc::ConnectionConfig> creds_;
|
||||
daemon::EmbeddedDaemon::OwnershipRecord recorded_; // pid > 0: our earlier session's detached node
|
||||
size_t credIndex_ = 0;
|
||||
rpc::RPCClient client_;
|
||||
bool clientOpen_ = false;
|
||||
@@ -5571,12 +5581,15 @@ void App::chooseStopAction(daemon::StopUserChoice choice)
|
||||
daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode, StopScope scope)
|
||||
{
|
||||
if (scope == StopScope::Policy) {
|
||||
if (!daemon_controller_) return daemon::StopOutcome::Exited;
|
||||
auto shutdownDecision = daemon_controller_->shutdownDecision(
|
||||
false, settings_ && settings_->getStopExternalDaemon());
|
||||
if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) {
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s — skipping\n", shutdownDecision.logReason);
|
||||
return daemon::StopOutcome::StillRunning;
|
||||
const bool mayStopExternal = mayStopExternalNode();
|
||||
if (!daemon_controller_) {
|
||||
if (!mayStopExternal) return daemon::StopOutcome::Exited; // nothing we may stop
|
||||
} else {
|
||||
auto shutdownDecision = daemon_controller_->shutdownDecision(false, mayStopExternal);
|
||||
if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) {
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s — skipping\n", shutdownDecision.logReason);
|
||||
return daemon::StopOutcome::StillRunning;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5590,6 +5603,11 @@ daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode, StopScope scope)
|
||||
// external node and a node we already let go of are not, and are never waited on, signalled or killed.
|
||||
const bool owned = d && d->beginStop();
|
||||
if (d && !owned) d->endStop(/*keepWatching=*/false);
|
||||
// Not launched by this session, but by an earlier one that left it running: we wait for that exact
|
||||
// process like our own (no time limit when quitting), but still never signal or kill it.
|
||||
daemon::EmbeddedDaemon::OwnershipRecord recorded;
|
||||
const bool ours = !owned && daemon::EmbeddedDaemon::liveOwnershipRecord(
|
||||
daemon::EmbeddedDaemon::ownershipRecordPath(), &recorded);
|
||||
|
||||
if (!owned && !daemon::EmbeddedDaemon::isRpcPortInUse()) {
|
||||
// Nothing listening for a stop: either it is gone, or it is another dragonxd we can't talk to (one
|
||||
@@ -5624,7 +5642,7 @@ daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode, StopScope scope)
|
||||
switch (mode) {
|
||||
case StopMode::AppShutdown:
|
||||
policy.userChoices = true;
|
||||
policy.maxWaitSec = owned ? 0.0 : 120.0; // a node we don't own: watch it go for up to 2 min
|
||||
policy.maxWaitSec = (owned || ours) ? 0.0 : 120.0; // a node we don't own: watch it go for up to 2 min
|
||||
break;
|
||||
case StopMode::Background:
|
||||
policy.maxWaitSec = 120.0;
|
||||
@@ -5637,15 +5655,19 @@ daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode, StopScope scope)
|
||||
break;
|
||||
}
|
||||
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s node, %s\n", owned ? "our own" : "an external",
|
||||
mode == StopMode::AppShutdown ? "quitting (no time limit; the user may leave it running)"
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s node, %s\n",
|
||||
owned ? "our own" : ours ? "our earlier session's (left running)" : "an external",
|
||||
mode == StopMode::AppShutdown
|
||||
? (policy.maxWaitSec > 0 ? "quitting (up to 120s for a node we don't own; never killed)"
|
||||
: "quitting (no time limit; the user may leave it running)")
|
||||
: mode == StopMode::Background ? "background (up to 120s, never killed)"
|
||||
: "fallback (up to 20s, never killed)");
|
||||
stop_kill_offered_ = false;
|
||||
stop_session_active_ = true;
|
||||
daemon::StopOutcome outcome;
|
||||
{
|
||||
AppStopRuntime runtime(*this, owned ? d : nullptr, std::move(creds));
|
||||
AppStopRuntime runtime(*this, owned ? d : nullptr, std::move(creds),
|
||||
ours ? recorded : daemon::EmbeddedDaemon::OwnershipRecord{});
|
||||
AppStopTaskContext task(*this, policy.abandonOnShutdown);
|
||||
outcome = daemon::DaemonController::executeDaemonStop(policy, runtime, task);
|
||||
}
|
||||
@@ -5654,6 +5676,8 @@ daemon::StopOutcome App::stopEmbeddedDaemon(StopMode mode, StopScope scope)
|
||||
// Still running after a background wait: keep watching it (its exit is booked as a clean stop). Left
|
||||
// running on purpose ("Leave running"), exited or killed: let go of it.
|
||||
if (owned) d->endStop(/*keepWatching=*/outcome == daemon::StopOutcome::StillRunning);
|
||||
if (outcome == daemon::StopOutcome::Exited || outcome == daemon::StopOutcome::Killed)
|
||||
daemon::EmbeddedDaemon::liveOwnershipRecord(daemon::EmbeddedDaemon::ownershipRecordPath()); // drops it if gone
|
||||
DEBUG_LOGF("stopEmbeddedDaemon: %s\n",
|
||||
outcome == daemon::StopOutcome::Exited ? "the node exited"
|
||||
: outcome == daemon::StopOutcome::Detached ? "left running at the user's request"
|
||||
@@ -6015,10 +6039,11 @@ void App::beginShutdown()
|
||||
DEBUG_LOGF("beginShutdown: starting (daemon_controller_=%s)\n",
|
||||
daemon_controller_ ? "yes" : "no");
|
||||
|
||||
// If no embedded daemon, just mark done — don't stop
|
||||
// an externally-managed daemon that the user started themselves.
|
||||
// Worker join + RPC disconnect happen in shutdown().
|
||||
if (!daemon_controller_) {
|
||||
// No controller (this session never launched a node): disconnect only — unless the node is the one our
|
||||
// earlier session left running, or stop_external_daemon is on. Worker join + RPC disconnect happen in
|
||||
// shutdown().
|
||||
const bool keepRunning = (settings_ && settings_->getKeepDaemonRunning()) || shutdown_keep_daemon_override_;
|
||||
if (!daemon_controller_ && (keepRunning || !mayStopExternalNode())) {
|
||||
DEBUG_LOGF("beginShutdown: no embedded daemon, disconnecting only\n");
|
||||
shutdown_status_ = TR("appx_disconnecting");
|
||||
if (settings_) {
|
||||
@@ -6033,17 +6058,18 @@ void App::beginShutdown()
|
||||
settings_->save();
|
||||
}
|
||||
|
||||
auto shutdownDecision = daemon_controller_->shutdownDecision(
|
||||
(settings_ && settings_->getKeepDaemonRunning()) || shutdown_keep_daemon_override_,
|
||||
settings_ && settings_->getStopExternalDaemon());
|
||||
if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) {
|
||||
DEBUG_LOGF("beginShutdown: %s, skipping daemon stop\n", shutdownDecision.logReason);
|
||||
if (daemon_controller_) {
|
||||
auto shutdownDecision = daemon_controller_->shutdownDecision(keepRunning, mayStopExternalNode());
|
||||
if (shutdownDecision.action == daemon::DaemonController::ShutdownAction::DisconnectOnly) {
|
||||
DEBUG_LOGF("beginShutdown: %s, skipping daemon stop\n", shutdownDecision.logReason);
|
||||
shutdown_status_ = shutdownDecision.status;
|
||||
shutdown_complete_ = true;
|
||||
return;
|
||||
}
|
||||
shutdown_status_ = shutdownDecision.status;
|
||||
shutdown_complete_ = true;
|
||||
return;
|
||||
} else {
|
||||
shutdown_status_ = TR("appx_sending_stop_command_to_daemon");
|
||||
}
|
||||
|
||||
shutdown_status_ = shutdownDecision.status;
|
||||
DEBUG_LOGF("beginShutdown: spawning shutdown thread for daemon stop\n");
|
||||
|
||||
// Run the daemon shutdown on a background thread so the UI
|
||||
@@ -6119,8 +6145,8 @@ static std::string formatBusyDuration(double secs)
|
||||
App::NodeBusyText App::nodeBusyText() const
|
||||
{
|
||||
NodeBusyText out;
|
||||
// The tracker reads OUR node's output; once that process is gone its last state means nothing.
|
||||
const bool ours = daemon_controller_ && daemon_controller_->isRunning();
|
||||
// Once the node the tracker follows is gone, its last state means nothing.
|
||||
const bool ours = busyTrackerIsLive();
|
||||
const bool rebuilding = ours && daemon_busy_.rebuilding();
|
||||
const bool replaying = ours && daemon_busy_.replaying() && !rebuilding;
|
||||
const bool rpcBusy = rpc_busy_gate_->busy();
|
||||
@@ -6169,6 +6195,87 @@ App::NodeBusyText App::nodeBusyText() const
|
||||
return out;
|
||||
}
|
||||
|
||||
bool App::busyTrackerIsLive() const
|
||||
{
|
||||
return (daemon_controller_ && daemon_controller_->isRunning()) || ext_log_tied_;
|
||||
}
|
||||
|
||||
long App::externalNodePid() const
|
||||
{
|
||||
daemon::EmbeddedDaemon::OwnershipRecord rec;
|
||||
if (daemon::EmbeddedDaemon::liveOwnershipRecord(daemon::EmbeddedDaemon::ownershipRecordPath(), &rec))
|
||||
return rec.pid;
|
||||
#ifndef _WIN32
|
||||
// POSIX dragonxd writes <datadir>/hushd.pid (removed on a clean exit): a live dragonxd under that pid uses
|
||||
// this datadir. (Windows writes none; there only our own record ties a node to this log.)
|
||||
std::ifstream in((std::filesystem::path(util::Platform::getDragonXDataDir()) / "hushd.pid").string());
|
||||
long pid = 0;
|
||||
if (in >> pid && pid > 0 && daemon::EmbeddedDaemon::processIsDaemon(pid)) return pid;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
void App::pumpExternalDaemonLog()
|
||||
{
|
||||
if (!supportsEmbeddedDaemon()) return;
|
||||
const double now = ImGui::GetTime();
|
||||
if (now < ext_log_next_at_) return;
|
||||
ext_log_next_at_ = now + 2.0;
|
||||
|
||||
auto untie = [this] {
|
||||
if (!ext_log_tied_) return;
|
||||
ext_log_tied_ = false;
|
||||
ext_log_pid_ = 0;
|
||||
daemon_busy_.reset();
|
||||
node_rebuilding_ = false;
|
||||
busy_first_seen_ = {};
|
||||
DEBUG_LOGF("[busy] stopped following the external node's debug.log\n");
|
||||
};
|
||||
auto readRange = [](const std::string& path, std::uintmax_t from, std::uintmax_t to) {
|
||||
std::string s;
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
if (!in || to <= from) return s;
|
||||
in.seekg(static_cast<std::streamoff>(from));
|
||||
s.resize(static_cast<size_t>(to - from));
|
||||
in.read(&s[0], static_cast<std::streamsize>(s.size()));
|
||||
s.resize(static_cast<size_t>(in.gcount()));
|
||||
return s;
|
||||
};
|
||||
|
||||
if (daemon_controller_ && daemon_controller_->isRunning()) { untie(); return; } // ours feeds it directly
|
||||
const long pid = externalNodePid();
|
||||
if (pid <= 0) { untie(); return; }
|
||||
const std::string path = (std::filesystem::path(util::Platform::getDragonXDataDir()) / "debug.log").string();
|
||||
std::error_code ec;
|
||||
const std::uintmax_t size = std::filesystem::file_size(path, ec);
|
||||
if (ec) { untie(); return; }
|
||||
|
||||
if (!ext_log_tied_ || pid != ext_log_pid_ || size < ext_log_offset_) {
|
||||
// (Re)tie: this process's lines from the last 2 MB, into a fresh tracker. The banner may be older than
|
||||
// the window on a long-running node; a rebuild still in progress logged its markers recently.
|
||||
constexpr std::uintmax_t kWindow = 2u * 1024u * 1024u;
|
||||
const std::uintmax_t from = size > kWindow ? size - kWindow : 0;
|
||||
const std::string window = readRange(path, from, size);
|
||||
const size_t at = daemon::externalLogFeedOffset(window, from > 0,
|
||||
daemon::EmbeddedDaemon::processStartUtc(pid));
|
||||
untie();
|
||||
if (at == std::string::npos) return; // not that process's log (it may log to its console)
|
||||
daemon_busy_.reset();
|
||||
ext_log_tied_ = true;
|
||||
ext_log_pid_ = pid;
|
||||
ext_log_offset_ = from + window.size();
|
||||
DEBUG_LOGF("[busy] following the debug.log of a node this session didn't launch (pid %ld)\n", pid);
|
||||
feedDaemonBusy(window.substr(at));
|
||||
return;
|
||||
}
|
||||
if (size > ext_log_offset_) {
|
||||
const std::uintmax_t to = std::min<std::uintmax_t>(size, ext_log_offset_ + 1024u * 1024u);
|
||||
const std::string chunk = readRange(path, ext_log_offset_, to);
|
||||
ext_log_offset_ += chunk.size();
|
||||
feedDaemonBusy(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
void App::resetDaemonBusy()
|
||||
{
|
||||
// Any thread (a restart/rescan task starts the node): the UI thread applies it before its next feed.
|
||||
@@ -6197,15 +6304,24 @@ bool App::rpcBusyWhileAlive() const
|
||||
return (daemon_controller_ && daemon_controller_->isRunning()) || daemon::EmbeddedDaemon::isRpcPortInUse();
|
||||
}
|
||||
|
||||
bool App::mayStopExternalNode() const
|
||||
{
|
||||
if (settings_ && settings_->getStopExternalDaemon()) return true;
|
||||
return daemon::EmbeddedDaemon::liveOwnershipRecord(daemon::EmbeddedDaemon::ownershipRecordPath());
|
||||
}
|
||||
|
||||
bool App::shouldConfirmDaemonStop() const
|
||||
{
|
||||
if (!daemon_controller_) return false;
|
||||
// Only relevant when this shutdown would actually STOP the daemon (embedded, or external with
|
||||
// stop-on-exit) — a DisconnectOnly shutdown leaves it running and loses nothing.
|
||||
const auto decision = daemon_controller_->shutdownDecision(
|
||||
settings_ && settings_->getKeepDaemonRunning(),
|
||||
settings_ && settings_->getStopExternalDaemon());
|
||||
const bool stops = decision.action == daemon::DaemonController::ShutdownAction::StopDaemon;
|
||||
// Only relevant when this shutdown would actually STOP the daemon (ours, ours left running by an
|
||||
// earlier session, or external with stop-on-exit) — a DisconnectOnly shutdown loses nothing.
|
||||
const bool keep = settings_ && settings_->getKeepDaemonRunning();
|
||||
bool stops = false;
|
||||
if (!daemon_controller_) {
|
||||
stops = !keep && mayStopExternalNode();
|
||||
} else {
|
||||
const auto decision = daemon_controller_->shutdownDecision(keep, mayStopExternalNode());
|
||||
stops = decision.action == daemon::DaemonController::ShutdownAction::StopDaemon;
|
||||
}
|
||||
// No daemon version suppresses this: v1.3.0 does not checkpoint a witness rebuild, and nothing survives
|
||||
// a kill.
|
||||
return stops && nodeBusyForStop();
|
||||
@@ -6216,7 +6332,7 @@ bool App::nodeBusyForStop() const
|
||||
// The tracker reads OUR node's output; once that process is gone its last state means nothing. An
|
||||
// external node's busyness shows only as RPC that has stopped answering.
|
||||
daemon::DaemonBusyTracker ours;
|
||||
if (daemon_controller_ && daemon_controller_->isRunning()) ours = daemon_busy_;
|
||||
if (busyTrackerIsLive()) ours = daemon_busy_;
|
||||
return daemon::shouldConfirmStop(true, ours, rpcBusyWhileAlive());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user