17ca2b0e69
/home/dev/lightwalletd/monitor_lwd.sh was carrying two uncommitted
production fixes, and the copy on disk had been reverted to the broken
committed version. The monitor that is actually running was executing a
deleted inode, so the fixes survived only as long as that process did --
any restart would have picked up the broken file.
The two fixes that were nearly lost:
* `-cache-size 5000` on the launch line. Without it a relaunch warms
the block cache from tip-400000 instead of tip-5000, which is
several minutes of getblock storm against the local node and several
minutes during which every wallet errors "Server's latest block is
behind ours".
* `EXIT_CODE=0; wait "$LWD_PID" || EXIT_CODE=$?` instead of
`wait "$LWD_PID" || true; EXIT_CODE=$?`. The latter reads the status
of `|| true` and is therefore always 0, so the monitor logged
"exited cleanly. Not restarting." and broke its loop on every exit
including crashes. That bug produced an 11h48m outage on 2026-08-21.
Recovered byte-identical from the running monitor via /proc/<pid>/fd/255
(md5 1823440d0af509c92583796af075b657) and committed so a checkout
cannot discard it again. An out-of-repo copy is kept at
/home/dev/monitor_lwd.sh.good.
Note the other branches still carry the broken blob; checking one out in
this working tree will clobber this file again. This working tree is a
live operational directory, not just a source checkout.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FU87LdsJZiZkfq1eXubpeo
127 lines
3.4 KiB
Bash
Executable File
127 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2024-2026 The DragonX Developers
|
|
# Released under GPLv3
|
|
#
|
|
# Monitors lightwalletd and restarts it automatically if it crashes.
|
|
# Usage: ./monitor_lwd.sh &
|
|
# or: nohup ./monitor_lwd.sh >> /tmp/lwd-monitor.log 2>&1 &
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
LWD_BIN="$SCRIPT_DIR/lightwalletd"
|
|
LWD_ARGS="-bind-addr lite.dragonx.is:9069 -conf-file $HOME/.hush/DRAGONX/DRAGONX.conf -no-tls -lag-min 4 -lag-max 12 -lag-window 30 -cache-size 5000"
|
|
LOGFILE="$SCRIPT_DIR/lwd-monitor.log"
|
|
PIDFILE="/tmp/lwd-monitor.pid"
|
|
STOPPING=0
|
|
RESTART_DELAY=5 # seconds to wait before restarting after a crash
|
|
MAX_RAPID_RESTARTS=5 # max restarts within the rapid window before backing off
|
|
RAPID_WINDOW=120 # seconds — if this many restarts happen within this window, back off
|
|
BACKOFF_DELAY=60 # seconds to wait when backing off
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
log() {
|
|
echo -e "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOGFILE"
|
|
}
|
|
|
|
cleanup() {
|
|
STOPPING=1
|
|
log "${YELLOW}Monitor shutting down...${NC}"
|
|
if [[ -n "${LWD_PID:-}" ]] && kill -0 "$LWD_PID" 2>/dev/null; then
|
|
log "Stopping lightwalletd (PID $LWD_PID)..."
|
|
kill "$LWD_PID" 2>/dev/null || true
|
|
wait "$LWD_PID" 2>/dev/null || true
|
|
fi
|
|
rm -f "$PIDFILE"
|
|
log "Monitor stopped."
|
|
exit 0
|
|
}
|
|
|
|
trap cleanup SIGINT SIGTERM
|
|
|
|
# Prevent duplicate monitors
|
|
if [[ -f "$PIDFILE" ]]; then
|
|
OLD_PID=$(cat "$PIDFILE")
|
|
if kill -0 "$OLD_PID" 2>/dev/null; then
|
|
echo "Monitor already running (PID $OLD_PID). Exiting."
|
|
exit 1
|
|
fi
|
|
rm -f "$PIDFILE"
|
|
fi
|
|
echo $$ > "$PIDFILE"
|
|
|
|
# Check binary exists
|
|
if [[ ! -x "$LWD_BIN" ]]; then
|
|
log "${RED}ERROR: lightwalletd binary not found at $LWD_BIN${NC}"
|
|
log "Build it first with: make build"
|
|
rm -f "$PIDFILE"
|
|
exit 1
|
|
fi
|
|
|
|
# Check conf file exists
|
|
CONF_FILE="$HOME/.hush/DRAGONX/DRAGONX.conf"
|
|
if [[ ! -f "$CONF_FILE" ]]; then
|
|
log "${RED}ERROR: DRAGONX.conf not found at $CONF_FILE${NC}"
|
|
rm -f "$PIDFILE"
|
|
exit 1
|
|
fi
|
|
|
|
log "${GREEN}DragonX lightwalletd monitor started (PID $$)${NC}"
|
|
log "Binary: $LWD_BIN"
|
|
log "Args: $LWD_ARGS"
|
|
|
|
restart_times=()
|
|
LWD_PID=""
|
|
STOPPING=0
|
|
|
|
while true; do
|
|
# Start lightwalletd
|
|
log "${GREEN}Starting lightwalletd...${NC}"
|
|
$LWD_BIN $LWD_ARGS >> "$LOGFILE" 2>&1 &
|
|
LWD_PID=$!
|
|
log "lightwalletd started with PID $LWD_PID"
|
|
|
|
# Wait for it to exit
|
|
EXIT_CODE=0
|
|
wait "$LWD_PID" || EXIT_CODE=$?
|
|
LWD_PID=""
|
|
|
|
if [[ $STOPPING -eq 1 ]]; then
|
|
log "${YELLOW}lightwalletd stopped on request (code $EXIT_CODE). Not restarting.${NC}"
|
|
break
|
|
fi
|
|
|
|
log "${RED}lightwalletd crashed with exit code $EXIT_CODE${NC}"
|
|
|
|
# Track restart frequency for backoff
|
|
NOW=$(date +%s)
|
|
restart_times+=("$NOW")
|
|
|
|
# Trim old entries outside the rapid window
|
|
CUTOFF=$((NOW - RAPID_WINDOW))
|
|
filtered=()
|
|
for t in "${restart_times[@]}"; do
|
|
if (( t >= CUTOFF )); then
|
|
filtered+=("$t")
|
|
fi
|
|
done
|
|
restart_times=("${filtered[@]}")
|
|
|
|
if (( ${#restart_times[@]} >= MAX_RAPID_RESTARTS )); then
|
|
log "${YELLOW}Too many restarts (${#restart_times[@]} in ${RAPID_WINDOW}s). Backing off for ${BACKOFF_DELAY}s...${NC}"
|
|
sleep "$BACKOFF_DELAY"
|
|
restart_times=()
|
|
else
|
|
log "Restarting in ${RESTART_DELAY}s..."
|
|
sleep "$RESTART_DELAY"
|
|
fi
|
|
done
|
|
|
|
rm -f "$PIDFILE"
|
|
log "Monitor exiting."
|