diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..1b15d40 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,40 @@ +# deploy/ + +Operational scripts, versioned here but **executed from outside this repository**. + +A node's runtime directory (`/home/dev/lightwalletd`) holds only the binary and +its logs. The supervisor scripts live in `/home/dev/`: + +| repo (source of truth) | deployed to | invoked by | +|------------------------|--------------------|---------------------------| +| `deploy/monitor_lwd.sh`| `/home/dev/monitor_lwd.sh` | `/home/dev/lwd_watchdog.sh` (cron, primary) | + +## Why they are not run from here + +This repository's working tree used to *be* the runtime directory, and +`monitor_lwd.sh` — the primary's live supervisor — was a tracked file inside it. +On 2026-08-26 a routine `git reset --hard` during an unrelated cherry-pick +reverted it to an older committed revision that (a) dropped `-cache-size 5000`, +making every relaunch warm the block cache from tip-400000, and (b) reintroduced +`wait "$LWD_PID" || true; EXIT_CODE=$?`, which reads the exit status of `|| true` +and is therefore always 0, so the monitor logged "exited cleanly. Not restarting." +and broke its loop on every exit including crashes — the bug behind an 11h48m +outage on 2026-08-21. + +Nothing noticed at the time because the running monitor was executing an +already-deleted inode: the working copy was broken while the live process was +fine. It was recovered from `/proc//fd/255`. + +Deploying these from outside the working tree means no checkout, reset, rebase or +branch switch can reach a running supervisor. + +## Changing one + +Edit it here, commit, then copy to the node and let the next relaunch pick it up: + + cp deploy/monitor_lwd.sh /home/dev/monitor_lwd.sh.stage + chmod 755 /home/dev/monitor_lwd.sh.stage + mv -f /home/dev/monitor_lwd.sh.stage /home/dev/monitor_lwd.sh + +`mv`, not `cp`: a rename cannot disturb a running process, and the currently +running monitor keeps its own inode until it next restarts. diff --git a/monitor_lwd.sh b/deploy/monitor_lwd.sh similarity index 84% rename from monitor_lwd.sh rename to deploy/monitor_lwd.sh index 1848e39..18d81c7 100755 --- a/monitor_lwd.sh +++ b/deploy/monitor_lwd.sh @@ -8,10 +8,17 @@ set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -LWD_BIN="$SCRIPT_DIR/lightwalletd" +# RUNTIME DIR IS EXPLICIT, not derived from this script's own location. +# This script used to live inside /home/dev/lightwalletd, which is a git +# working tree as well as the runtime directory -- so a routine `git reset +# --hard` or branch checkout there silently reverted the live supervisor to an +# older committed version. That happened on 2026-08-26; the running monitor +# survived only because it was executing an already-deleted inode. The script +# now lives outside the repo and names the runtime dir directly. +LWD_DIR="${LWD_DIR:-/home/dev/lightwalletd}" +LWD_BIN="$LWD_DIR/lightwalletd" LWD_ARGS="-bind-addr lite.dragonx.is:9069 -conf-file $HOME/.hush/DRAGONX/DRAGONX.conf -no-tls -lag-min 4 -lag-max 12 -lag-window 30 -cache-size 5000" -LOGFILE="$SCRIPT_DIR/lwd-monitor.log" +LOGFILE="$LWD_DIR/lwd-monitor.log" PIDFILE="/tmp/lwd-monitor.pid" STOPPING=0 RESTART_DELAY=5 # seconds to wait before restarting after a crash