diff --git a/src/daemon/embedded_daemon.cpp b/src/daemon/embedded_daemon.cpp index 0e83df9..71a0f3a 100644 --- a/src/daemon/embedded_daemon.cpp +++ b/src/daemon/embedded_daemon.cpp @@ -64,7 +64,7 @@ EmbeddedDaemon::~EmbeddedDaemon() // process (seed migration's isolated node) could take down the user's main node. Owners call stop() // explicitly (the app's own daemon is never destroyed: main() ends in _Exit). Just stop watching and // release what we hold; a process still running is left to finish on its own. - should_stop_ = true; + requestMonitorStop(); if (monitor_thread_.joinable()) { monitor_thread_.join(); } @@ -254,6 +254,24 @@ std::vector EmbeddedDaemon::getChainParams() }; } +void EmbeddedDaemon::requestMonitorStop() +{ + // Take monitor_mutex_ so the flag cannot flip between the monitor's predicate check and its wait + // (lost wakeup). Lock order lifecycle_mutex_ -> monitor_mutex_ is deadlock-free: the monitor thread + // never takes lifecycle_mutex_. + { + std::lock_guard lk(monitor_mutex_); + should_stop_ = true; + } + monitor_cv_.notify_all(); +} + +size_t EmbeddedDaemon::outputTotalSnapshot() const +{ + std::lock_guard lk(output_mutex_); + return output_total_; +} + void EmbeddedDaemon::setState(State s, const std::string& message) { // Message first, state second: a reader that sees Error must also see that error's message, not the @@ -512,7 +530,7 @@ bool EmbeddedDaemon::start(const std::string& binary_path) // still joinable. Join it, then release what its run left behind. Only in this case: if a stop session // already joined it, the fd/handle belong to that session and endStop() closes them. if (monitor_thread_.joinable()) { - should_stop_ = true; // a monitor somehow still looping ends within one tick + requestMonitorStop(); // a monitor somehow still looping wakes immediately via monitor_cv_ monitor_thread_.join(); // normally immediate (it already returned) #ifdef _WIN32 if (process_handle_ != nullptr) { CloseHandle(process_handle_); process_handle_ = nullptr; } @@ -713,7 +731,7 @@ bool EmbeddedDaemon::start(const std::string& binary_path) // Supervise the process just launched. A monitor left over from an earlier run was joined above. should_stop_ = false; if (monitor_thread_.joinable()) { // unreachable under lifecycle_mutex_; never join a live monitor unsignalled - should_stop_ = true; monitor_thread_.join(); should_stop_ = false; + requestMonitorStop(); monitor_thread_.join(); should_stop_ = false; } monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this); @@ -947,7 +965,7 @@ bool EmbeddedDaemon::stop(int wait_ms) { std::lock_guard life(lifecycle_mutex_); stop_requested_ = true; - should_stop_ = true; + requestMonitorStop(); } setState(State::Stopping, "Stopping dragonxd..."); @@ -1021,7 +1039,7 @@ bool EmbeddedDaemon::beginStop() { std::lock_guard life(lifecycle_mutex_); // before the flags: a racing start() would reset should_stop_ stop_requested_ = true; - should_stop_ = true; + requestMonitorStop(); if (monitor_thread_.joinable()) { monitor_thread_.join(); } @@ -1073,7 +1091,7 @@ void EmbeddedDaemon::endStop(bool keepWatching) if (keepWatching && isRunning()) { // Still running and still ours: watch it again. Its exit is booked as a clean stop. if (monitor_thread_.joinable()) { // unreachable after beginStop()'s join; never join a live monitor unsignalled - should_stop_ = true; monitor_thread_.join(); + requestMonitorStop(); monitor_thread_.join(); } should_stop_ = false; monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this); @@ -1116,10 +1134,23 @@ static std::string translateWindowsExitCode(DWORD code) { void EmbeddedDaemon::monitorProcess() { + const auto fast = std::chrono::milliseconds(250); + const auto cap = std::chrono::milliseconds(500); + const auto startTime = std::chrono::steady_clock::now(); + auto lastData = startTime; + auto interval = fast; while (!should_stop_ && isRunning()) { // Tail debug.log for new output + const size_t before = outputTotalSnapshot(); drainOutput(); - std::this_thread::sleep_for(std::chrono::milliseconds(250)); + const auto now = std::chrono::steady_clock::now(); + const bool got = outputTotalSnapshot() != before; + if (got) lastData = now; + interval = nextMonitorInterval(interval, got, + std::chrono::duration_cast(now - startTime), + std::chrono::duration_cast(now - lastData), fast, cap); + std::unique_lock lk(monitor_mutex_); + monitor_cv_.wait_for(lk, interval, [this] { return should_stop_.load(); }); } // Read any remaining output after process exits @@ -1537,7 +1568,7 @@ bool EmbeddedDaemon::stop(int wait_ms) { bool exitedOnItsOwn = true; - // Join the monitor FIRST (it sees should_stop_ within one 100ms tick) so that from here on this is + // Join the monitor FIRST (it wakes immediately via monitor_cv_) so that from here on this is // the only reaper. isRunning() is state-based and stays true until we set Stopped below, so it can't // tell us the child exited: poll waitpid() on a cached pid instead. If the child exited inside that // tick, the monitor already reaped it and, seeing stop_requested_, recorded a clean stop (pid now 0). @@ -1545,7 +1576,7 @@ bool EmbeddedDaemon::stop(int wait_ms) { std::lock_guard life(lifecycle_mutex_); stop_requested_ = true; - should_stop_ = true; + requestMonitorStop(); if (monitor_thread_.joinable()) { monitor_thread_.join(); } @@ -1594,8 +1625,8 @@ bool EmbeddedDaemon::beginStop() { std::lock_guard life(lifecycle_mutex_); // before the flags: a racing start() would reset should_stop_ stop_requested_ = true; - should_stop_ = true; - // The monitor sees should_stop_ within one 100ms tick. If the child exited in that window the + requestMonitorStop(); + // The monitor wakes immediately via monitor_cv_. If the child exited in that window the // monitor reaped it (as a clean stop: stop_requested_ is set) and the pid is now 0. if (monitor_thread_.joinable()) { monitor_thread_.join(); @@ -1690,7 +1721,7 @@ void EmbeddedDaemon::endStop(bool keepWatching) if (pid > 0 && keepWatching) { // Still running and still ours: watch it again. Its exit is booked as a clean stop. if (monitor_thread_.joinable()) { // unreachable after beginStop()'s join; never join a live monitor unsignalled - should_stop_ = true; monitor_thread_.join(); + requestMonitorStop(); monitor_thread_.join(); } should_stop_ = false; monitor_thread_ = std::thread(&EmbeddedDaemon::monitorProcess, this); @@ -1716,6 +1747,11 @@ void EmbeddedDaemon::endStop(bool keepWatching) void EmbeddedDaemon::monitorProcess() { + const auto fast = std::chrono::milliseconds(100); + const auto cap = std::chrono::milliseconds(500); + const auto startTime = std::chrono::steady_clock::now(); + auto lastData = startTime; + auto interval = fast; while (!should_stop_) { // Check if process exited (non-blocking waitpid captures exit status) if (process_pid_ > 0) { @@ -1742,9 +1778,18 @@ void EmbeddedDaemon::monitorProcess() } // Read process output (non-blocking): the stdout/stderr pipe and the node's debug.log + const size_t before = outputTotalSnapshot(); drainOutput(); + const auto now = std::chrono::steady_clock::now(); + const bool got = outputTotalSnapshot() != before; + if (got) lastData = now; + interval = nextMonitorInterval(interval, got, + std::chrono::duration_cast(now - startTime), + std::chrono::duration_cast(now - lastData), fast, cap); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + // Wakes immediately on requestMonitorStop(); idle polling backs off to `cap`. + std::unique_lock lk(monitor_mutex_); + monitor_cv_.wait_for(lk, interval, [this] { return should_stop_.load(); }); } } diff --git a/src/daemon/embedded_daemon.h b/src/daemon/embedded_daemon.h index 47be8b1..f979a56 100644 --- a/src/daemon/embedded_daemon.h +++ b/src/daemon/embedded_daemon.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include @@ -474,9 +476,23 @@ public: * reader can tell a new crash from an error it has already reported. */ std::uint64_t getUnexpectedExitCount() const { return unexpected_exits_.load(); } + /** Monitor poll interval: stay `fast` for the first 30 s after the thread starts and for 3 s after any + * output; otherwise double per idle tick up to `cap`. Any output resets to `fast`. Pure. */ + static std::chrono::milliseconds nextMonitorInterval(std::chrono::milliseconds cur, bool bytesRead, + std::chrono::milliseconds sinceStart, std::chrono::milliseconds sinceLastData, + std::chrono::milliseconds fast, std::chrono::milliseconds cap) + { + using ms = std::chrono::milliseconds; + if (bytesRead || sinceStart < ms(30000) || sinceLastData < ms(3000)) return fast; + const ms next = cur * 2; + return next > cap ? cap : (next < fast ? fast : next); + } + private: void setState(State s, const std::string& message = ""); void monitorProcess(); + void requestMonitorStop(); // set should_stop_ under monitor_mutex_ and wake the monitor + size_t outputTotalSnapshot() const; // output_total_ under output_mutex_ (monitor idle detection) void drainOutput(); // read any pending stdout into process_output_ void appendOutput(const char* data, size_t len); // append + trim (caller holds output_mutex_) bool startProcess(const std::string& binary_path, const std::vector& args); @@ -514,6 +530,8 @@ private: std::thread monitor_thread_; std::atomic should_stop_{false}; + std::mutex monitor_mutex_; // pairs with monitor_cv_ so stop wakes the monitor at once + std::condition_variable monitor_cv_; std::atomic stop_requested_{false}; // a stop was asked for: the next exit is not a crash // Serializes start/beginStop/stop/endStop over monitor_thread_, should_stop_ and stop_requested_ (two // threads joining one std::thread is undefined). Never taken by the monitor thread or the state callback. diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp index 3a10869..cea39ab 100644 --- a/tests/test_phase4.cpp +++ b/tests/test_phase4.cpp @@ -6071,6 +6071,24 @@ void testOutputSinceAcrossTrim() EXPECT_TRUE(got.empty()); } +void testNextMonitorInterval() +{ + using dragonx::daemon::EmbeddedDaemon; + using ms = std::chrono::milliseconds; + const ms fast(100), cap(500); + auto next = [&](ms cur, bool got, ms sinceStart, ms sinceData) { + return EmbeddedDaemon::nextMonitorInterval(cur, got, sinceStart, sinceData, fast, cap); + }; + EXPECT_TRUE(next(fast, false, ms(1000), ms(1000)) == fast); // first 30 s stay fast + EXPECT_TRUE(next(cap, false, ms(29999), ms(60000)) == fast); + EXPECT_TRUE(next(cap, false, ms(60000), ms(2999)) == fast); // recent data stays fast + EXPECT_TRUE(next(cap, true, ms(60000), ms(60000)) == fast); // data resets + EXPECT_TRUE(next(fast, false, ms(60000), ms(10000)) == ms(200)); // doubling + EXPECT_TRUE(next(ms(200), false, ms(60000), ms(10000)) == ms(400)); + EXPECT_TRUE(next(ms(400), false, ms(60000), ms(10000)) == cap); // capped at 500 + EXPECT_TRUE(next(cap, false, ms(60000), ms(10000)) == cap); +} + void testOutputWindowSaturation() { using dragonx::daemon::appendBounded; @@ -18694,6 +18712,7 @@ int main() testShouldConfirmStop(); testOutputSinceAcrossTrim(); testOutputWindowSaturation(); + testNextMonitorInterval(); testOutputEndWatermarkReplay(); testChooseDaemonStatus(); testDaemonStopStringsTranslated();