4b5a5e5a67
/home/dev/lightwalletd is simultaneously a git checkout and the runtime directory, and monitor_lwd.sh -- the primary's live supervisor -- was a tracked file inside it. So a routine `git reset --hard` during an unrelated cherry-pick reverted the running supervisor to an older committed revision. That happened on 2026-08-26. Nothing noticed, because the running monitor was executing an already-deleted inode: the file on disk was broken while the live process was fine. It was recovered from /proc/<pid>/fd/255. What the reverted copy would have reintroduced, had it ever restarted: the loss of `-cache-size 5000`, so every relaunch warms the block cache from tip-400000 instead of tip-5000; and `wait "$LWD_PID" || true; EXIT_CODE=$?`, which reads the exit status of `|| true` and is therefore always 0, so the monitor logs "exited cleanly. Not restarting." and breaks its loop on every exit including crashes -- the bug behind an 11h48m outage on 2026-08-21. Move it to deploy/, from where it is copied to /home/dev/ and run. The runtime directory now holds only the binary and its logs, so no checkout, reset, rebase or branch switch can reach a running supervisor. The script no longer derives its paths from its own location: SCRIPT_DIR became an explicit LWD_DIR, because the script and the runtime directory are deliberately no longer the same place. lwd_watchdog.sh launches "$MONITOR" by absolute path and refuses to run if it is missing. Verified: run from /tmp, the relocated monitor resolves its binary and log through LWD_DIR rather than its own directory, and launches with -cache-size 5000 intact. The running monitor was not restarted. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FU87LdsJZiZkfq1eXubpeo
134 lines
3.9 KiB
Bash
Executable File
134 lines
3.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2024-2026 The DragonX Developers
|
|
# Released under GPLv3
|
|
#
|
|
# Monitors lightwalletd and restarts it automatically if it crashes.
|
|
# Usage: ./monitor_lwd.sh &
|
|
# or: nohup ./monitor_lwd.sh >> /tmp/lwd-monitor.log 2>&1 &
|
|
|
|
set -euo pipefail
|
|
|
|
# RUNTIME DIR IS EXPLICIT, not derived from this script's own location.
|
|
# This script used to live inside /home/dev/lightwalletd, which is a git
|
|
# working tree as well as the runtime directory -- so a routine `git reset
|
|
# --hard` or branch checkout there silently reverted the live supervisor to an
|
|
# older committed version. That happened on 2026-08-26; the running monitor
|
|
# survived only because it was executing an already-deleted inode. The script
|
|
# now lives outside the repo and names the runtime dir directly.
|
|
LWD_DIR="${LWD_DIR:-/home/dev/lightwalletd}"
|
|
LWD_BIN="$LWD_DIR/lightwalletd"
|
|
LWD_ARGS="-bind-addr lite.dragonx.is:9069 -conf-file $HOME/.hush/DRAGONX/DRAGONX.conf -no-tls -lag-min 4 -lag-max 12 -lag-window 30 -cache-size 5000"
|
|
LOGFILE="$LWD_DIR/lwd-monitor.log"
|
|
PIDFILE="/tmp/lwd-monitor.pid"
|
|
STOPPING=0
|
|
RESTART_DELAY=5 # seconds to wait before restarting after a crash
|
|
MAX_RAPID_RESTARTS=5 # max restarts within the rapid window before backing off
|
|
RAPID_WINDOW=120 # seconds — if this many restarts happen within this window, back off
|
|
BACKOFF_DELAY=60 # seconds to wait when backing off
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
log() {
|
|
echo -e "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOGFILE"
|
|
}
|
|
|
|
cleanup() {
|
|
STOPPING=1
|
|
log "${YELLOW}Monitor shutting down...${NC}"
|
|
if [[ -n "${LWD_PID:-}" ]] && kill -0 "$LWD_PID" 2>/dev/null; then
|
|
log "Stopping lightwalletd (PID $LWD_PID)..."
|
|
kill "$LWD_PID" 2>/dev/null || true
|
|
wait "$LWD_PID" 2>/dev/null || true
|
|
fi
|
|
rm -f "$PIDFILE"
|
|
log "Monitor stopped."
|
|
exit 0
|
|
}
|
|
|
|
trap cleanup SIGINT SIGTERM
|
|
|
|
# Prevent duplicate monitors
|
|
if [[ -f "$PIDFILE" ]]; then
|
|
OLD_PID=$(cat "$PIDFILE")
|
|
if kill -0 "$OLD_PID" 2>/dev/null; then
|
|
echo "Monitor already running (PID $OLD_PID). Exiting."
|
|
exit 1
|
|
fi
|
|
rm -f "$PIDFILE"
|
|
fi
|
|
echo $$ > "$PIDFILE"
|
|
|
|
# Check binary exists
|
|
if [[ ! -x "$LWD_BIN" ]]; then
|
|
log "${RED}ERROR: lightwalletd binary not found at $LWD_BIN${NC}"
|
|
log "Build it first with: make build"
|
|
rm -f "$PIDFILE"
|
|
exit 1
|
|
fi
|
|
|
|
# Check conf file exists
|
|
CONF_FILE="$HOME/.hush/DRAGONX/DRAGONX.conf"
|
|
if [[ ! -f "$CONF_FILE" ]]; then
|
|
log "${RED}ERROR: DRAGONX.conf not found at $CONF_FILE${NC}"
|
|
rm -f "$PIDFILE"
|
|
exit 1
|
|
fi
|
|
|
|
log "${GREEN}DragonX lightwalletd monitor started (PID $$)${NC}"
|
|
log "Binary: $LWD_BIN"
|
|
log "Args: $LWD_ARGS"
|
|
|
|
restart_times=()
|
|
LWD_PID=""
|
|
STOPPING=0
|
|
|
|
while true; do
|
|
# Start lightwalletd
|
|
log "${GREEN}Starting lightwalletd...${NC}"
|
|
$LWD_BIN $LWD_ARGS >> "$LOGFILE" 2>&1 &
|
|
LWD_PID=$!
|
|
log "lightwalletd started with PID $LWD_PID"
|
|
|
|
# Wait for it to exit
|
|
EXIT_CODE=0
|
|
wait "$LWD_PID" || EXIT_CODE=$?
|
|
LWD_PID=""
|
|
|
|
if [[ $STOPPING -eq 1 ]]; then
|
|
log "${YELLOW}lightwalletd stopped on request (code $EXIT_CODE). Not restarting.${NC}"
|
|
break
|
|
fi
|
|
|
|
log "${RED}lightwalletd crashed with exit code $EXIT_CODE${NC}"
|
|
|
|
# Track restart frequency for backoff
|
|
NOW=$(date +%s)
|
|
restart_times+=("$NOW")
|
|
|
|
# Trim old entries outside the rapid window
|
|
CUTOFF=$((NOW - RAPID_WINDOW))
|
|
filtered=()
|
|
for t in "${restart_times[@]}"; do
|
|
if (( t >= CUTOFF )); then
|
|
filtered+=("$t")
|
|
fi
|
|
done
|
|
restart_times=("${filtered[@]}")
|
|
|
|
if (( ${#restart_times[@]} >= MAX_RAPID_RESTARTS )); then
|
|
log "${YELLOW}Too many restarts (${#restart_times[@]} in ${RAPID_WINDOW}s). Backing off for ${BACKOFF_DELAY}s...${NC}"
|
|
sleep "$BACKOFF_DELAY"
|
|
restart_times=()
|
|
else
|
|
log "Restarting in ${RESTART_DELAY}s..."
|
|
sleep "$RESTART_DELAY"
|
|
fi
|
|
done
|
|
|
|
rm -f "$PIDFILE"
|
|
log "Monitor exiting."
|