perf(daemon): back off the monitor poll when idle

The daemon monitor thread drained output and then slept a fixed short
interval on every tick, waking several times a second for the life of
the node even long after startup, when the node emits almost nothing.
Stopping it also meant waiting out the current sleep before it saw the
stop flag.

- The monitor now waits on a condition variable instead of a plain
  sleep, so a stop request wakes it immediately rather than after the
  interval. The stop flag is set under the monitor's mutex so the wake
  cannot be lost.
- The poll interval stays fast for the first 30 s after the thread
  starts and for 3 s after any output, and otherwise doubles per idle
  tick up to a 500 ms cap. Any new output drops it straight back to
  fast, so responsiveness during startup, rescan and shutdown is
  unchanged while a quiet node is polled far less often. The interval
  choice is a pure function.

Tests cover the interval function across the warmup window, the
recent-data window, the reset on new output, the doubling, and the cap.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/daemon/embedded_daemon.cpp

@@ -64,7 +64,7 @@ EmbeddedDaemon::~EmbeddedDaemon()
     // process (seed migration's isolated node) could take down the user's main node. Owners call stop()
     // explicitly (the app's own daemon is never destroyed: main() ends in _Exit). Just stop watching and
     // release what we hold; a process still running is left to finish on its own.
-    should_stop_ = true;
+    requestMonitorStop();
     if (monitor_thread_.joinable()) {
         monitor_thread_.join();
     }
@@ -254,6 +254,24 @@ std::vector<std::string> EmbeddedDaemon::getChainParams()
     };
 }
 
+void EmbeddedDaemon::requestMonitorStop()
+{
+    // Take monitor_mutex_ so the flag cannot flip between the monitor's predicate check and its wait
+    // (lost wakeup). Lock order lifecycle_mutex_ -> monitor_mutex_ is deadlock-free: the monitor thread
+    // never takes lifecycle_mutex_.
+    {
+        std::lock_guard<std::mutex> lk(monitor_mutex_);
+        should_stop_ = true;
+    }
+    monitor_cv_.notify_all();
+}
+
+size_t EmbeddedDaemon::outputTotalSnapshot() const
+{
+    std::lock_guard<std::mutex> lk(output_mutex_);
+    return output_total_;
+}
+
 void EmbeddedDaemon::setState(State s, const std::string& message)
 {
     // Message first, state second: a reader that sees Error must also see that error's message, not the
@@ -512,7 +530,7 @@ bool EmbeddedDaemon::start(const std::string& binary_path)
     // still joinable. Join it, then release what its run left behind. Only in this case: if a stop session
     // already joined it, the fd/handle belong to that session and endStop() closes them.
     if (monitor_thread_.joinable()) {
-        should_stop_ = true;               // a monitor somehow still looping ends within one tick
+        requestMonitorStop();               // a monitor somehow still looping wakes immediately via monitor_cv_
         monitor_thread_.join();            // normally immediate (it already returned)
 #ifdef _WIN32
         if (process_handle_ != nullptr) { CloseHandle(process_handle_); process_handle_ = nullptr; }
@@ -713,7 +731,7 @@ bool EmbeddedDaemon::start(const std::string& binary_path)
     // Supervise the process just launched. A monitor left over from an earlier run was joined above.
     should_stop_ = false;
     if (monitor_thread_.joinable()) {      // unreachable under lifecycle_mutex_; never join a live monitor unsignalled
-        should_stop_ = true; monitor_thread_.join(); should_stop_ = false;
+        requestMonitorStop(); monitor_thread_.join(); should_stop_ = false;
     }
     monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
     
@@ -947,7 +965,7 @@ bool EmbeddedDaemon::stop(int wait_ms)
     {
         std::lock_guard<std::mutex> life(lifecycle_mutex_);
         stop_requested_ = true;
-        should_stop_ = true;
+        requestMonitorStop();
     }
     setState(State::Stopping, "Stopping dragonxd...");
     
@@ -1021,7 +1039,7 @@ bool EmbeddedDaemon::beginStop()
 {
     std::lock_guard<std::mutex> life(lifecycle_mutex_);   // before the flags: a racing start() would reset should_stop_
     stop_requested_ = true;
-    should_stop_ = true;
+    requestMonitorStop();
     if (monitor_thread_.joinable()) {
         monitor_thread_.join();
     }
@@ -1073,7 +1091,7 @@ void EmbeddedDaemon::endStop(bool keepWatching)
     if (keepWatching && isRunning()) {
         // Still running and still ours: watch it again. Its exit is booked as a clean stop.
         if (monitor_thread_.joinable()) {   // unreachable after beginStop()'s join; never join a live monitor unsignalled
-            should_stop_ = true; monitor_thread_.join();
+            requestMonitorStop(); monitor_thread_.join();
         }
         should_stop_ = false;
         monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
@@ -1116,10 +1134,23 @@ static std::string translateWindowsExitCode(DWORD code) {
 
 void EmbeddedDaemon::monitorProcess()
 {
+    const auto fast = std::chrono::milliseconds(250);
+    const auto cap = std::chrono::milliseconds(500);
+    const auto startTime = std::chrono::steady_clock::now();
+    auto lastData = startTime;
+    auto interval = fast;
     while (!should_stop_ && isRunning()) {
         // Tail debug.log for new output
+        const size_t before = outputTotalSnapshot();
         drainOutput();
-        std::this_thread::sleep_for(std::chrono::milliseconds(250));
+        const auto now = std::chrono::steady_clock::now();
+        const bool got = outputTotalSnapshot() != before;
+        if (got) lastData = now;
+        interval = nextMonitorInterval(interval, got,
+            std::chrono::duration_cast<std::chrono::milliseconds>(now - startTime),
+            std::chrono::duration_cast<std::chrono::milliseconds>(now - lastData), fast, cap);
+        std::unique_lock<std::mutex> lk(monitor_mutex_);
+        monitor_cv_.wait_for(lk, interval, [this] { return should_stop_.load(); });
     }
     
     // Read any remaining output after process exits
@@ -1537,7 +1568,7 @@ bool EmbeddedDaemon::stop(int wait_ms)
 {
     bool exitedOnItsOwn = true;
 
-    // Join the monitor FIRST (it sees should_stop_ within one 100ms tick) so that from here on this is
+    // Join the monitor FIRST (it wakes immediately via monitor_cv_) so that from here on this is
     // the only reaper. isRunning() is state-based and stays true until we set Stopped below, so it can't
     // tell us the child exited: poll waitpid() on a cached pid instead. If the child exited inside that
     // tick, the monitor already reaped it and, seeing stop_requested_, recorded a clean stop (pid now 0).
@@ -1545,7 +1576,7 @@ bool EmbeddedDaemon::stop(int wait_ms)
     {
         std::lock_guard<std::mutex> life(lifecycle_mutex_);
         stop_requested_ = true;
-        should_stop_ = true;
+        requestMonitorStop();
         if (monitor_thread_.joinable()) {
             monitor_thread_.join();
         }
@@ -1594,8 +1625,8 @@ bool EmbeddedDaemon::beginStop()
 {
     std::lock_guard<std::mutex> life(lifecycle_mutex_);   // before the flags: a racing start() would reset should_stop_
     stop_requested_ = true;
-    should_stop_ = true;
-    // The monitor sees should_stop_ within one 100ms tick. If the child exited in that window the
+    requestMonitorStop();
+    // The monitor wakes immediately via monitor_cv_. If the child exited in that window the
     // monitor reaped it (as a clean stop: stop_requested_ is set) and the pid is now 0.
     if (monitor_thread_.joinable()) {
         monitor_thread_.join();
@@ -1690,7 +1721,7 @@ void EmbeddedDaemon::endStop(bool keepWatching)
     if (pid > 0 && keepWatching) {
         // Still running and still ours: watch it again. Its exit is booked as a clean stop.
         if (monitor_thread_.joinable()) {   // unreachable after beginStop()'s join; never join a live monitor unsignalled
-            should_stop_ = true; monitor_thread_.join();
+            requestMonitorStop(); monitor_thread_.join();
         }
         should_stop_ = false;
         monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this);
@@ -1716,6 +1747,11 @@ void EmbeddedDaemon::endStop(bool keepWatching)
 
 void EmbeddedDaemon::monitorProcess()
 {
+    const auto fast = std::chrono::milliseconds(100);
+    const auto cap = std::chrono::milliseconds(500);
+    const auto startTime = std::chrono::steady_clock::now();
+    auto lastData = startTime;
+    auto interval = fast;
     while (!should_stop_) {
         // Check if process exited (non-blocking waitpid captures exit status)
         if (process_pid_ > 0) {
@@ -1742,9 +1778,18 @@ void EmbeddedDaemon::monitorProcess()
         }
 
         // Read process output (non-blocking): the stdout/stderr pipe and the node's debug.log
+        const size_t before = outputTotalSnapshot();
         drainOutput();
+        const auto now = std::chrono::steady_clock::now();
+        const bool got = outputTotalSnapshot() != before;
+        if (got) lastData = now;
+        interval = nextMonitorInterval(interval, got,
+            std::chrono::duration_cast<std::chrono::milliseconds>(now - startTime),
+            std::chrono::duration_cast<std::chrono::milliseconds>(now - lastData), fast, cap);
 
-        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        // Wakes immediately on requestMonitorStop(); idle polling backs off to `cap`.
+        std::unique_lock<std::mutex> lk(monitor_mutex_);
+        monitor_cv_.wait_for(lk, interval, [this] { return should_stop_.load(); });
     }
 }
 

src/daemon/embedded_daemon.h

@@ -14,6 +14,8 @@
 #include <atomic>
 #include <thread>
 #include <mutex>
+#include <condition_variable>
+#include <chrono>
 #include <set>
 #include <vector>
 
@@ -474,9 +476,23 @@ public:
      *  reader can tell a new crash from an error it has already reported. */
     std::uint64_t getUnexpectedExitCount() const { return unexpected_exits_.load(); }
 
+    /** Monitor poll interval: stay `fast` for the first 30 s after the thread starts and for 3 s after any
+     *  output; otherwise double per idle tick up to `cap`. Any output resets to `fast`. Pure. */
+    static std::chrono::milliseconds nextMonitorInterval(std::chrono::milliseconds cur, bool bytesRead,
+            std::chrono::milliseconds sinceStart, std::chrono::milliseconds sinceLastData,
+            std::chrono::milliseconds fast, std::chrono::milliseconds cap)
+    {
+        using ms = std::chrono::milliseconds;
+        if (bytesRead || sinceStart < ms(30000) || sinceLastData < ms(3000)) return fast;
+        const ms next = cur * 2;
+        return next > cap ? cap : (next < fast ? fast : next);
+    }
+
 private:
     void setState(State s, const std::string& message = "");
     void monitorProcess();
+    void requestMonitorStop();                 // set should_stop_ under monitor_mutex_ and wake the monitor
+    size_t outputTotalSnapshot() const;        // output_total_ under output_mutex_ (monitor idle detection)
     void drainOutput();   // read any pending stdout into process_output_
     void appendOutput(const char* data, size_t len); // append + trim (caller holds output_mutex_)
     bool startProcess(const std::string& binary_path, const std::vector<std::string>& args);
@@ -514,6 +530,8 @@ private:
 
     std::thread monitor_thread_;
     std::atomic<bool> should_stop_{false};
+    std::mutex monitor_mutex_;                 // pairs with monitor_cv_ so stop wakes the monitor at once
+    std::condition_variable monitor_cv_;
     std::atomic<bool> stop_requested_{false};  // a stop was asked for: the next exit is not a crash
     // Serializes start/beginStop/stop/endStop over monitor_thread_, should_stop_ and stop_requested_ (two
     // threads joining one std::thread is undefined). Never taken by the monitor thread or the state callback.

tests/test_phase4.cpp

@@ -6071,6 +6071,24 @@ void testOutputSinceAcrossTrim()
     EXPECT_TRUE(got.empty());
 }
 
+void testNextMonitorInterval()
+{
+    using dragonx::daemon::EmbeddedDaemon;
+    using ms = std::chrono::milliseconds;
+    const ms fast(100), cap(500);
+    auto next = [&](ms cur, bool got, ms sinceStart, ms sinceData) {
+        return EmbeddedDaemon::nextMonitorInterval(cur, got, sinceStart, sinceData, fast, cap);
+    };
+    EXPECT_TRUE(next(fast, false, ms(1000), ms(1000)) == fast);        // first 30 s stay fast
+    EXPECT_TRUE(next(cap, false, ms(29999), ms(60000)) == fast);
+    EXPECT_TRUE(next(cap, false, ms(60000), ms(2999)) == fast);        // recent data stays fast
+    EXPECT_TRUE(next(cap, true, ms(60000), ms(60000)) == fast);        // data resets
+    EXPECT_TRUE(next(fast, false, ms(60000), ms(10000)) == ms(200));   // doubling
+    EXPECT_TRUE(next(ms(200), false, ms(60000), ms(10000)) == ms(400));
+    EXPECT_TRUE(next(ms(400), false, ms(60000), ms(10000)) == cap);    // capped at 500
+    EXPECT_TRUE(next(cap, false, ms(60000), ms(10000)) == cap);
+}
+
 void testOutputWindowSaturation()
 {
     using dragonx::daemon::appendBounded;
@@ -18694,6 +18712,7 @@ int main()
     testShouldConfirmStop();
     testOutputSinceAcrossTrim();
     testOutputWindowSaturation();
+    testNextMonitorInterval();
     testOutputEndWatermarkReplay();
     testChooseDaemonStatus();
     testDaemonStopStringsTranslated();