Dai Ha
3ba6d6784c
Adds scripts/bridged-launchd-wrapper.sh so the launchd-run daemon still gets WORKER_GITEA_TOKEN/AI_GATEWAY_TOKEN by execing through a login shell (launchd never sources secrets.sh itself). bridged now logs at startup which required token env vars (derived from each profile's tokenEnv/gitTokenEnv, not a hand-written list) resolved or are MISSING, by name only. Fills in the real paths in deploy/dev.ltms.bridged.plist for this host and points it at the wrapper. scripts/redeploy-bridged.sh now detects a loaded launchd agent and uses launchctl unload/load instead of a raw kill+nohup, because a bare SIGTERM exits this JVM at 143 (measured) which KeepAlive.SuccessfulExit=false reads as a crash and would race the script's own restart; --check reports installed/loaded state and stays read-only.
297 lines
14 KiB
Bash
Executable File
297 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Rebuild and restart the bridged daemon.
|
|
#
|
|
# A merge is not a deployment: the running daemon holds the jar it was started with, so code merged
|
|
# to main does nothing until this runs. See CLAUDE.md -> "Redeploying the daemon".
|
|
#
|
|
# This script exists to turn six remembered traps into one auditable command:
|
|
#
|
|
# 1. A piped `mvn` hides BUILD FAILURE behind a zero exit, so the build here is never piped.
|
|
# 2. The daemon must start from a LOGIN shell, or the tokens it hands to members are empty:
|
|
# WORKER_GITEA_TOKEN (workers cannot open a PR) and AI_GATEWAY_TOKEN (401 at llm.ltms.dev).
|
|
# Both are read from the DAEMON's own environment at spawn time, so a value added to
|
|
# secrets.sh after startup is absent. Nothing logs this here, so the script checks and says
|
|
# so — and since CB-594, bridged's own startup log says so too, by env var name.
|
|
# 3. An old daemon that never actually died looks identical from the outside, so the script waits
|
|
# for the process to exit and for the port to free before it starts a new one.
|
|
# 4. "It started" is not "it works": the script polls /healthz until it answers, and reports the
|
|
# herdr protocol number, because healthz can be green while every spawn fails on a protocol
|
|
# mismatch.
|
|
# 5. Restarting under live members drops their tickets, so the script refuses unless you confirm
|
|
# the fleet is drained.
|
|
# 6. CB-594 — the launchd agent (deploy/dev.ltms.bridged.plist), if installed and loaded, is a
|
|
# SECOND supervisor: its KeepAlive.SuccessfulExit=false restarts the daemon on any nonzero
|
|
# exit, and a bare SIGTERM makes this JVM exit 143 even with its shutdown hook running to
|
|
# completion (measured — see the CB-594 report). A plain `kill` here would race launchd's own
|
|
# restart of the OLD jar. So this script detects whether the agent is loaded and, only then,
|
|
# swaps `kill` + manual `nohup` for `launchctl unload`/`load` — the one supervisor in control
|
|
# at any moment is whichever one you asked to act, never both.
|
|
#
|
|
# Usage:
|
|
# scripts/redeploy-bridged.sh # build, confirm, restart, verify
|
|
# scripts/redeploy-bridged.sh --yes # skip the drain confirmation (fleet already checked)
|
|
# scripts/redeploy-bridged.sh --no-build # restart the jar already on disk
|
|
# scripts/redeploy-bridged.sh --check # report state and exit; changes nothing
|
|
#
|
|
# Exits non-zero on any failure. A failed build never stops the running daemon.
|
|
|
|
set -euo pipefail
|
|
|
|
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
BRIDGED="$REPO/bridged"
|
|
JAR="$BRIDGED/target/bridged.jar"
|
|
OUT="$BRIDGED/bridged.out"
|
|
# Matches BOTH the absolute form and the relative `java -jar target/bridged.jar` a hand-start
|
|
# produces from inside bridged/. Anchoring on the absolute path alone was a real bug: the daemon
|
|
# restarted correctly and the script still reported "no process appeared", because it launched with
|
|
# a relative path and then looked for an absolute one.
|
|
PATTERN='target/bridged.jar'
|
|
HEALTH='http://127.0.0.1:8765/healthz'
|
|
STOP_WAIT=30 # seconds to wait for a clean exit before reporting failure
|
|
HEALTH_WAIT=60 # seconds to wait for /healthz to answer after start
|
|
|
|
# CB-594: the launchd agent this script must not fight with (see trap 6 above).
|
|
LAUNCHD_LABEL='dev.ltms.bridged'
|
|
LAUNCHD_PLIST="$HOME/Library/LaunchAgents/$LAUNCHD_LABEL.plist"
|
|
|
|
DO_BUILD=1; ASSUME_YES=0; CHECK_ONLY=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--yes|-y) ASSUME_YES=1 ;;
|
|
--no-build) DO_BUILD=0 ;;
|
|
--check) CHECK_ONLY=1 ;;
|
|
-h|--help) sed -n '3,37p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
*) echo "unknown option: $arg (try --help)" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
say() { printf '\n\033[1m== %s\033[0m\n' "$*"; }
|
|
ok() { printf ' ok %s\n' "$*"; }
|
|
warn() { printf ' WARN %s\n' "$*"; }
|
|
die() { printf '\n FAIL %s\n\n' "$*" >&2; exit 1; }
|
|
|
|
jar_id() { [ -f "$JAR" ] && shasum -a 256 "$JAR" | cut -c1-12 || echo "absent"; }
|
|
running_pid() { pgrep -f "$PATTERN" || true; }
|
|
# `launchctl list <label>` exits 0 iff the label is loaded (registered with launchd) — true whether
|
|
# or not it is currently running, which is exactly "supervision is active" for our purposes. Read-
|
|
# only: neither helper below changes anything, so both are also safe under --check.
|
|
launchd_installed() { [ -f "$LAUNCHD_PLIST" ]; }
|
|
launchd_loaded() { launchctl list "$LAUNCHD_LABEL" >/dev/null 2>&1; }
|
|
|
|
# ---------------------------------------------------------------- report state
|
|
|
|
say "current state"
|
|
OLD_PID="$(running_pid)"
|
|
if [ -n "$OLD_PID" ]; then
|
|
ok "daemon running, pid $OLD_PID"
|
|
else
|
|
warn "no daemon running — this will be a cold start"
|
|
fi
|
|
ok "jar on disk: $(jar_id) ($([ -f "$JAR" ] && date -r "$JAR" '+%Y-%m-%d %H:%M:%S' || echo 'none'))"
|
|
ok "HEAD: $(git -C "$REPO" log --oneline -1)"
|
|
|
|
# CB-594: supervision state. Installed and loaded are different facts — a copied-but-never-loaded
|
|
# plist supervises nothing, and a loaded label with no file backing it (rare, but possible after an
|
|
# edited/moved plist) is still what launchd will act on.
|
|
if launchd_installed; then
|
|
ok "launchd agent installed: $LAUNCHD_PLIST"
|
|
else
|
|
warn "launchd agent NOT installed (no supervision — a crash will not restart the daemon)."
|
|
fi
|
|
SUPERVISED=0
|
|
if launchd_loaded; then
|
|
SUPERVISED=1
|
|
ok "launchd agent loaded ($LAUNCHD_LABEL) — launchd supervises this daemon"
|
|
else
|
|
warn "launchd agent not loaded — this script is the only thing that will restart the daemon."
|
|
fi
|
|
|
|
# The trap with no log line. Checked in a LOGIN shell, because that is how the daemon is started
|
|
# below. Never prints the value — only whether it resolved.
|
|
if zsh -lc '[ -n "${WORKER_GITEA_TOKEN:-}" ]' 2>/dev/null; then
|
|
ok "WORKER_GITEA_TOKEN resolves in a login shell"
|
|
else
|
|
warn "WORKER_GITEA_TOKEN is EMPTY in a login shell."
|
|
warn "The daemon will start fine and workers will silently fail to open PRs."
|
|
warn "Fix \${SHARED_ENV}/tools/secrets.sh before relying on worker checkpoints."
|
|
fi
|
|
|
|
# Same trap, second variable (CB-591). A profile's `tokenEnv:` is resolved from the DAEMON's own
|
|
# process environment by HerdrPeerLauncher.resolveEnv, so a token added to secrets.sh after the
|
|
# daemon started is simply absent. The launcher then injects an empty token and llm.ltms.dev answers
|
|
# 401 — long after the restart, and with nothing tying the two together.
|
|
if zsh -lc '[ -n "${AI_GATEWAY_TOKEN:-}" ]' 2>/dev/null; then
|
|
ok "AI_GATEWAY_TOKEN resolves in a login shell"
|
|
else
|
|
warn "AI_GATEWAY_TOKEN is EMPTY in a login shell."
|
|
warn "Any profile whose tokenEnv is AI_GATEWAY_TOKEN will get an empty token and 401 at the gateway."
|
|
warn "This only matters once a profile points at llm.ltms.dev — harmless before that."
|
|
fi
|
|
|
|
if [ "$CHECK_ONLY" = 1 ]; then
|
|
say "--check: nothing changed"
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------- build
|
|
# Deliberately before the stop: a failed build must never leave the fleet down.
|
|
|
|
if [ "$DO_BUILD" = 1 ]; then
|
|
say "build"
|
|
BUILD_LOG="$(mktemp -t bridged-build)"
|
|
echo " log: $BUILD_LOG"
|
|
if ! mvn -f "$BRIDGED/pom.xml" clean install > "$BUILD_LOG" 2>&1; then
|
|
grep -E 'ERROR|BUILD FAILURE|Tests run:.*Failures: [1-9]|Tests run:.*Errors: [1-9]' "$BUILD_LOG" \
|
|
| head -20 || true
|
|
die "build failed — the running daemon was NOT touched. Full log: $BUILD_LOG"
|
|
fi
|
|
grep -E '^\[INFO\] Tests run:.*Failures' "$BUILD_LOG" | tail -1 | sed 's/^\[INFO\] / /' || true
|
|
ok "BUILD SUCCESS"
|
|
ok "jar now: $(jar_id)"
|
|
else
|
|
say "build skipped (--no-build)"
|
|
fi
|
|
|
|
[ -f "$JAR" ] || die "no jar at $JAR — run without --no-build"
|
|
|
|
# ----------------------------------------------------------------- drain gate
|
|
|
|
if [ -n "$OLD_PID" ] && [ "$ASSUME_YES" = 0 ]; then
|
|
say "drain check"
|
|
echo " A restart drops every in-flight ticket and rendezvous. A member's report"
|
|
echo " is NOT recoverable once its ticket is gone."
|
|
echo
|
|
echo " Confirm with bridge_list that no members are live, and bridge_poll anything"
|
|
echo " you still want, BEFORE continuing."
|
|
echo
|
|
read -r -p " Fleet drained? type yes to restart: " reply
|
|
[ "$reply" = "yes" ] || die "aborted — nothing changed"
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ stop
|
|
#
|
|
# CB-594: when SUPERVISED, launchd owns the stop — never a raw `kill` here. A bare SIGTERM makes
|
|
# this JVM exit 143 even with its shutdown hook running to completion (verified separately: a
|
|
# throwaway Java process with an equivalent shutdown hook, sent SIGTERM from a login shell that
|
|
# could `wait` on it directly, reported exit code 143 every time — never 0). launchd's
|
|
# KeepAlive.SuccessfulExit=false treats any nonzero exit as a crash and restarts the OLD jar,
|
|
# which would race this script's own restart of the NEW one. `launchctl unload` avoids that race
|
|
# by deregistering the job first, so no KeepAlive is left armed when the process actually stops.
|
|
|
|
if [ -n "$OLD_PID" ]; then
|
|
say "stop"
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)" # verify a FRESH line appears later
|
|
if [ "$SUPERVISED" = 1 ]; then
|
|
echo " supervision is ON: using 'launchctl unload' (not kill) so launchd's own KeepAlive"
|
|
echo " cannot restart the OLD jar out from under this script — see the CB-594 comment above."
|
|
launchctl unload -w "$LAUNCHD_PLIST" \
|
|
|| die "launchctl unload failed — the daemon may still be under supervision; investigate before retrying"
|
|
else
|
|
kill "$OLD_PID"
|
|
fi
|
|
for _ in $(seq "$STOP_WAIT"); do
|
|
[ -z "$(running_pid)" ] && break
|
|
sleep 1
|
|
done
|
|
if [ -n "$(running_pid)" ]; then
|
|
die "pid $OLD_PID still alive after ${STOP_WAIT}s. Not escalating to kill -9 automatically:
|
|
the shutdown hook releases sessions and worktrees in order, and killing it hard can
|
|
leave worktrees and panes behind. Investigate, then kill -9 by hand if you accept that."
|
|
fi
|
|
ok "pid $OLD_PID exited"
|
|
elif [ "$SUPERVISED" = 1 ]; then
|
|
# Loaded but not currently running (e.g. throttled after a crash loop). Unload it anyway so the
|
|
# start step below does a clean load, never a load stacked on an already-loaded label.
|
|
say "stop"
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)"
|
|
launchctl unload -w "$LAUNCHD_PLIST" 2>/dev/null || true
|
|
ok "launchd agent unloaded (was already not running)"
|
|
else
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)"
|
|
fi
|
|
|
|
# ------------------------------------------------------------------ start
|
|
# Unsupervised: login shell (zsh -l) is what puts the secrets on the daemon's environment, and cwd
|
|
# must be bridged/ because the daemon resolves bridged.yaml, logs/ and target/ relative to it.
|
|
# Supervised: launchd does both — deploy/dev.ltms.bridged.plist points ProgramArguments at
|
|
# scripts/bridged-launchd-wrapper.sh (CB-594), which is what execs the login shell in launchd's
|
|
# place, and WorkingDirectory in the plist already pins bridged/.
|
|
|
|
say "start"
|
|
if [ "$SUPERVISED" = 1 ]; then
|
|
echo " supervision is ON: using 'launchctl load' so launchd starts and keeps supervising this"
|
|
echo " process, instead of a manual nohup that launchd would know nothing about."
|
|
launchctl load -w "$LAUNCHD_PLIST" || die "launchctl load failed"
|
|
else
|
|
# Absolute jar path so `ps` names which checkout is running.
|
|
( cd "$BRIDGED" && zsh -lc "nohup java -jar '$JAR' >> bridged.out 2>&1 &" )
|
|
fi
|
|
|
|
for _ in $(seq 10); do
|
|
NEW_PID="$(running_pid)"
|
|
[ -n "$NEW_PID" ] && break
|
|
sleep 1
|
|
done
|
|
[ -n "${NEW_PID:-}" ] || die "no process appeared. Last lines of $OUT:
|
|
$(tail -20 "$OUT" 2>/dev/null)"
|
|
[ "$NEW_PID" != "${OLD_PID:-}" ] || die "pid unchanged ($NEW_PID) — the old daemon never died"
|
|
ok "started, pid $NEW_PID"
|
|
|
|
# ------------------------------------------------------------------ verify
|
|
|
|
say "verify"
|
|
|
|
HEALTH_BODY=""
|
|
for _ in $(seq "$HEALTH_WAIT"); do
|
|
if HEALTH_BODY="$(curl -fsS --max-time 2 "$HEALTH" 2>/dev/null)"; then break; fi
|
|
HEALTH_BODY=""
|
|
sleep 1
|
|
done
|
|
|
|
if [ -z "$HEALTH_BODY" ]; then
|
|
# 503 still means the daemon is up — it means herdr is unreachable. Say which.
|
|
CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 2 "$HEALTH" 2>/dev/null || echo 000)"
|
|
if [ "$CODE" = "503" ]; then
|
|
warn "/healthz answers 503 degraded — the daemon is up but herdr is unreachable."
|
|
warn "Spawns will fail. Check herdr before delegating anything."
|
|
curl -s --max-time 2 "$HEALTH" 2>/dev/null | head -3 || true
|
|
else
|
|
die "/healthz never answered within ${HEALTH_WAIT}s (last code: $CODE). Last lines of $OUT:
|
|
$(tail -30 "$OUT" 2>/dev/null)"
|
|
fi
|
|
else
|
|
ok "/healthz 200 — $HEALTH_BODY"
|
|
warn "healthz green only proves herdr ANSWERS. If its protocol number changed, spawns can still"
|
|
warn "fail — prove a real spawn before trusting the fleet."
|
|
fi
|
|
|
|
# A fresh listening line, strictly after the restart mark. An old daemon that never died would
|
|
# otherwise let an old line pass for a new one.
|
|
if tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null | grep -q 'bridged listening'; then
|
|
ok "$(tail -n "+$((RESTART_MARK + 1))" "$OUT" | grep 'bridged listening' | tail -1)"
|
|
else
|
|
warn "no fresh 'bridged listening' line after the restart — check $OUT yourself"
|
|
fi
|
|
|
|
# Config keys the daemon accepted or deferred at boot. This is usually WHY you restarted.
|
|
say "config at boot"
|
|
tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null \
|
|
| grep -iE 'deferred|classification:|fleet health:|coverage' | tail -8 | sed 's/^/ /' \
|
|
|| echo " (nothing reported)"
|
|
|
|
# Errors since the restart, anchored to the marker so old noise cannot leak in.
|
|
ERRS="$(tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null | grep -cE ' (ERROR|SEVERE) ' || true)"
|
|
say "result"
|
|
ok "pid $NEW_PID, jar $(jar_id)"
|
|
if [ "${ERRS:-0}" -gt 0 ]; then
|
|
warn "$ERRS ERROR lines since restart:"
|
|
tail -n "+$((RESTART_MARK + 1))" "$OUT" | grep -E ' (ERROR|SEVERE) ' | tail -5 | sed 's/^/ /'
|
|
else
|
|
ok "no ERROR lines since restart"
|
|
fi
|
|
echo
|
|
echo " Next: call bridge_whoami and confirm it still answers 'primary'. A lead whose tab label"
|
|
echo " no longer matches fleet.leaders.*.tab is demoted to worker and refuses orchestration."
|
|
echo
|