From 17ca2b0e69984f172e019a8003f7ae3a4efb2ef2 Mon Sep 17 00:00:00 2001 From: DragonX Developers Date: Wed, 26 Aug 2026 11:12:48 -0500 Subject: [PATCH] monitor: commit the running supervisor, which existed only in memory /home/dev/lightwalletd/monitor_lwd.sh was carrying two uncommitted production fixes, and the copy on disk had been reverted to the broken committed version. The monitor that is actually running was executing a deleted inode, so the fixes survived only as long as that process did -- any restart would have picked up the broken file. The two fixes that were nearly lost: * `-cache-size 5000` on the launch line. Without it a relaunch warms the block cache from tip-400000 instead of tip-5000, which is several minutes of getblock storm against the local node and several minutes during which every wallet errors "Server's latest block is behind ours". * `EXIT_CODE=0; wait "$LWD_PID" || EXIT_CODE=$?` instead of `wait "$LWD_PID" || true; EXIT_CODE=$?`. The latter reads the status of `|| true` and is therefore always 0, so the monitor logged "exited cleanly. Not restarting." and broke its loop on every exit including crashes. That bug produced an 11h48m outage on 2026-08-21. Recovered byte-identical from the running monitor via /proc//fd/255 (md5 1823440d0af509c92583796af075b657) and committed so a checkout cannot discard it again. An out-of-repo copy is kept at /home/dev/monitor_lwd.sh.good. Note the other branches still carry the broken blob; checking one out in this working tree will clobber this file again. This working tree is a live operational directory, not just a source checkout. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FU87LdsJZiZkfq1eXubpeo --- monitor_lwd.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/monitor_lwd.sh b/monitor_lwd.sh index fdb8c6f..1848e39 100755 --- a/monitor_lwd.sh +++ b/monitor_lwd.sh @@ -10,9 +10,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" LWD_BIN="$SCRIPT_DIR/lightwalletd" -LWD_ARGS="-bind-addr lite.dragonx.is:9069 -conf-file $HOME/.hush/DRAGONX/DRAGONX.conf -no-tls -lag-min 4 -lag-max 12 -lag-window 30" +LWD_ARGS="-bind-addr lite.dragonx.is:9069 -conf-file $HOME/.hush/DRAGONX/DRAGONX.conf -no-tls -lag-min 4 -lag-max 12 -lag-window 30 -cache-size 5000" LOGFILE="$SCRIPT_DIR/lwd-monitor.log" PIDFILE="/tmp/lwd-monitor.pid" +STOPPING=0 RESTART_DELAY=5 # seconds to wait before restarting after a crash MAX_RAPID_RESTARTS=5 # max restarts within the rapid window before backing off RAPID_WINDOW=120 # seconds — if this many restarts happen within this window, back off @@ -29,6 +30,7 @@ log() { } cleanup() { + STOPPING=1 log "${YELLOW}Monitor shutting down...${NC}" if [[ -n "${LWD_PID:-}" ]] && kill -0 "$LWD_PID" 2>/dev/null; then log "Stopping lightwalletd (PID $LWD_PID)..." @@ -75,6 +77,7 @@ log "Args: $LWD_ARGS" restart_times=() LWD_PID="" +STOPPING=0 while true; do # Start lightwalletd @@ -84,12 +87,12 @@ while true; do log "lightwalletd started with PID $LWD_PID" # Wait for it to exit - wait "$LWD_PID" || true - EXIT_CODE=$? + EXIT_CODE=0 + wait "$LWD_PID" || EXIT_CODE=$? LWD_PID="" - if [[ $EXIT_CODE -eq 0 ]]; then - log "${YELLOW}lightwalletd exited cleanly (code 0). Not restarting.${NC}" + if [[ $STOPPING -eq 1 ]]; then + log "${YELLOW}lightwalletd stopped on request (code $EXIT_CODE). Not restarting.${NC}" break fi