|
|
|
@@ -5,7 +5,7 @@
|
|
|
|
|
# A merge is not a deployment: the running daemon holds the jar it was started with, so code merged
|
|
|
|
|
# to main does nothing until this runs. See CLAUDE.md -> "Redeploying the daemon".
|
|
|
|
|
#
|
|
|
|
|
# This script exists to turn six remembered traps into one auditable command:
|
|
|
|
|
# This script exists to turn eight remembered traps into one auditable command:
|
|
|
|
|
#
|
|
|
|
|
# 1. A piped `mvn` hides BUILD FAILURE behind a zero exit, so the build here is never piped.
|
|
|
|
|
# 2. The daemon must start from a LOGIN shell, or the tokens it hands to members are empty:
|
|
|
|
@@ -27,6 +27,17 @@
|
|
|
|
|
# restart of the OLD jar. So this script detects whether the agent is loaded and, only then,
|
|
|
|
|
# swaps `kill` + manual `nohup` for `launchctl unload`/`load` — the one supervisor in control
|
|
|
|
|
# at any moment is whichever one you asked to act, never both.
|
|
|
|
|
# 7. fleetd #492 — a systemd --user unit is a THIRD possible supervisor (seen on a second host):
|
|
|
|
|
# Restart=on-failure treats this JVM's SIGTERM exit code (143, per CB-594 above) as a failure
|
|
|
|
|
# too, so a bare `kill` there would race systemd's own restart of the OLD jar exactly like
|
|
|
|
|
# launchd would. This script now tells launchd, systemd, and "genuinely unsupervised" apart as
|
|
|
|
|
# three different answers, drives whichever one it finds through its own control plane
|
|
|
|
|
# (`launchctl` / `systemctl --user`), and REFUSES outright — never falls back to `kill` — when
|
|
|
|
|
# it finds a supervision signal it cannot map to exactly one of the two it knows how to drive.
|
|
|
|
|
# A wrong guess here is how two daemons end up running against one herdr session.
|
|
|
|
|
# 8. fleetd #492 — a post-restart check counts running fleetd processes and fails the whole run if
|
|
|
|
|
# more than one is alive. That is the one thing none of the checks above (healthz 200, jar id,
|
|
|
|
|
# the fresh "listening" line) can see: every one of them is satisfied by EITHER daemon.
|
|
|
|
|
#
|
|
|
|
|
# Usage:
|
|
|
|
|
# scripts/redeploy-fleetd.sh # build, confirm, restart, verify
|
|
|
|
@@ -55,13 +66,18 @@ HEALTH_WAIT=60 # seconds to wait for /healthz to answer after start
|
|
|
|
|
LAUNCHD_LABEL='dev.ltms.fleetd'
|
|
|
|
|
LAUNCHD_PLIST="$HOME/Library/LaunchAgents/$LAUNCHD_LABEL.plist"
|
|
|
|
|
|
|
|
|
|
# fleetd #492: the systemd --user unit this script must not fight with either (see trap 7 above).
|
|
|
|
|
# Measured on the second host: `systemctl --user cat fleetd` names the unit "fleetd" (not
|
|
|
|
|
# "dev.ltms.fleetd" — systemd user units here are not namespaced the way the launchd label is).
|
|
|
|
|
SYSTEMD_UNIT='fleetd'
|
|
|
|
|
|
|
|
|
|
DO_BUILD=1; ASSUME_YES=0; CHECK_ONLY=0
|
|
|
|
|
for arg in "$@"; do
|
|
|
|
|
case "$arg" in
|
|
|
|
|
--yes|-y) ASSUME_YES=1 ;;
|
|
|
|
|
--no-build) DO_BUILD=0 ;;
|
|
|
|
|
--check) CHECK_ONLY=1 ;;
|
|
|
|
|
-h|--help) sed -n '3,37p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
|
|
|
-h|--help) sed -n '3,48p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
|
|
|
*) echo "unknown option: $arg (try --help)" >&2; exit 2 ;;
|
|
|
|
|
esac
|
|
|
|
|
done
|
|
|
|
@@ -79,6 +95,91 @@ running_pid() { pgrep -f "$PATTERN" || true; }
|
|
|
|
|
launchd_installed() { [ -f "$LAUNCHD_PLIST" ]; }
|
|
|
|
|
launchd_loaded() { launchctl list "$LAUNCHD_LABEL" >/dev/null 2>&1; }
|
|
|
|
|
|
|
|
|
|
# fleetd #492: same two questions for systemd --user. Kept as separate, overridable functions
|
|
|
|
|
# (never an inline `systemctl` call at each use site) so a test on a box with no systemd at all
|
|
|
|
|
# (this repo is developed on macOS) can substitute each one independently — the same seam
|
|
|
|
|
# launchd_installed/launchd_loaded above already use.
|
|
|
|
|
#
|
|
|
|
|
# "installed": a unit FILE by this name exists, regardless of its current state — the systemd
|
|
|
|
|
# analogue of the plist file existing on disk. `list-unit-files` reads unit definitions without
|
|
|
|
|
# depending on runtime state, so this stays read-only and safe under --check.
|
|
|
|
|
systemd_installed() {
|
|
|
|
|
command -v systemctl >/dev/null 2>&1 \
|
|
|
|
|
&& systemctl --user list-unit-files "$SYSTEMD_UNIT.service" --no-legend 2>/dev/null | grep -q .
|
|
|
|
|
}
|
|
|
|
|
# "loaded": systemd currently supervises this unit as an active job — the systemd analogue of
|
|
|
|
|
# `launchctl list <label>` succeeding. Measured on the second host: `systemctl --user is-active
|
|
|
|
|
# fleetd` -> "active".
|
|
|
|
|
systemd_loaded() {
|
|
|
|
|
command -v systemctl >/dev/null 2>&1 && systemctl --user is-active "$SYSTEMD_UNIT" >/dev/null 2>&1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# fleetd #492: three real answers, not two — launchd, systemd, or genuinely unsupervised — plus a
|
|
|
|
|
# fourth, "ambiguous", for the one case this script cannot tell apart: both signals firing at once.
|
|
|
|
|
# That is exactly "I cannot tell who supervises this process", and guessing wrong here is how two
|
|
|
|
|
# daemons end up running against one herdr session (see trap 7 in the header). Pure and
|
|
|
|
|
# side-effect-free: reads the two probes above and decides — never mutates anything, so it is safe
|
|
|
|
|
# under --check and testable by overriding launchd_loaded/systemd_loaded after sourcing.
|
|
|
|
|
detect_supervisor() {
|
|
|
|
|
local ld=0 sd=0
|
|
|
|
|
launchd_loaded && ld=1
|
|
|
|
|
systemd_loaded && sd=1
|
|
|
|
|
if [ "$ld" = 1 ] && [ "$sd" = 1 ]; then
|
|
|
|
|
echo "ambiguous"
|
|
|
|
|
elif [ "$ld" = 1 ]; then
|
|
|
|
|
echo "launchd"
|
|
|
|
|
elif [ "$sd" = 1 ]; then
|
|
|
|
|
echo "systemd"
|
|
|
|
|
else
|
|
|
|
|
echo "none"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# fleetd #492: turns anything detect_supervisor returns that is NOT exactly one of the two
|
|
|
|
|
# supervisors this script knows how to drive into a die() — never a fall-through to the `kill`
|
|
|
|
|
# path. Kept as its own function so a test can call it directly (in a subshell, since it die()s)
|
|
|
|
|
# without running the whole report-state flow or needing a real launchd/systemd.
|
|
|
|
|
require_drivable_supervisor() {
|
|
|
|
|
local kind="$1"
|
|
|
|
|
case "$kind" in
|
|
|
|
|
launchd|systemd|none) ;;
|
|
|
|
|
ambiguous)
|
|
|
|
|
die "both launchd ($LAUNCHD_LABEL) and systemd --user ($SYSTEMD_UNIT) report themselves as
|
|
|
|
|
loaded for this daemon at the same time. This script cannot tell which one actually
|
|
|
|
|
supervises the running process, and driving either alone risks the OTHER reviving the
|
|
|
|
|
OLD jar out from under it — the exact failure this ticket (fleetd #492) exists to
|
|
|
|
|
prevent. Stop one of the two supervisors by hand, confirm only one remains loaded, then
|
|
|
|
|
rerun." ;;
|
|
|
|
|
*)
|
|
|
|
|
die "detect_supervisor returned an unrecognized value '$kind' — refusing to guess which
|
|
|
|
|
supervisor, if any, controls this daemon." ;;
|
|
|
|
|
esac
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# fleetd #492: the exact symptom a racing supervisor produces — count how many fleetd processes are
|
|
|
|
|
# alive right now. Takes the pid list as a parameter (rather than calling running_pid() itself) so a
|
|
|
|
|
# test can pass a canned two-line string without a real second process running. Pure except for the
|
|
|
|
|
# die() in assert_single_daemon below.
|
|
|
|
|
count_daemon_pids() {
|
|
|
|
|
local pids="$1"
|
|
|
|
|
if [ -z "$pids" ]; then
|
|
|
|
|
echo 0
|
|
|
|
|
else
|
|
|
|
|
printf '%s\n' "$pids" | grep -c .
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
assert_single_daemon() {
|
|
|
|
|
local pids="$1" count
|
|
|
|
|
count="$(count_daemon_pids "$pids")"
|
|
|
|
|
if [ "$count" -gt 1 ]; then
|
|
|
|
|
die "more than one fleetd process is running after this restart (pids: $(printf '%s' "$pids" | tr '\n' ' ')).
|
|
|
|
|
This is the exact failure a racing supervisor produces: the OLD jar was revived by its
|
|
|
|
|
supervisor while this script started a NEW copy. Two daemons on one herdr session kill
|
|
|
|
|
each other's members. Investigate with 'pgrep -f \"$PATTERN\"' and stop the wrong one by
|
|
|
|
|
hand — do not assume either pid is the one you want."
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# CB-600: the script computes its own log path from where it sits on disk (REPO, above); the
|
|
|
|
|
# plist hard-codes an absolute StandardOutPath. Nothing forced the two to agree — if this script
|
|
|
|
|
# were ever run from a checkout other than the one the loaded plist names, launchd would start and
|
|
|
|
@@ -182,25 +283,45 @@ fi
|
|
|
|
|
ok "jar on disk: $(jar_id) ($([ -f "$JAR" ] && date -r "$JAR" '+%Y-%m-%d %H:%M:%S' || echo 'none'))"
|
|
|
|
|
ok "HEAD: $(git -C "$REPO" log --oneline -1)"
|
|
|
|
|
|
|
|
|
|
# CB-594: supervision state. Installed and loaded are different facts — a copied-but-never-loaded
|
|
|
|
|
# plist supervises nothing, and a loaded label with no file backing it (rare, but possible after an
|
|
|
|
|
# edited/moved plist) is still what launchd will act on.
|
|
|
|
|
# CB-594 / fleetd #492: supervision state. Installed and loaded are different facts — a
|
|
|
|
|
# copied-but-never-loaded plist (or an unloaded systemd unit) supervises nothing, and a loaded
|
|
|
|
|
# label/unit with no file backing it is still what its supervisor will act on.
|
|
|
|
|
if launchd_installed; then
|
|
|
|
|
ok "launchd agent installed: $LAUNCHD_PLIST"
|
|
|
|
|
else
|
|
|
|
|
warn "launchd agent NOT installed (no supervision — a crash will not restart the daemon)."
|
|
|
|
|
warn "launchd agent NOT installed."
|
|
|
|
|
fi
|
|
|
|
|
SUPERVISED=0
|
|
|
|
|
if launchd_loaded; then
|
|
|
|
|
SUPERVISED=1
|
|
|
|
|
ok "launchd agent loaded ($LAUNCHD_LABEL) — launchd supervises this daemon"
|
|
|
|
|
# CB-600: fail loudly here, before ANY other check runs, if this script and the loaded plist
|
|
|
|
|
# would read different log files — every check after this point is worthless otherwise.
|
|
|
|
|
check_log_path_matches_plist "$OUT" "$LAUNCHD_PLIST"
|
|
|
|
|
if systemd_installed; then
|
|
|
|
|
ok "systemd --user unit installed: $SYSTEMD_UNIT"
|
|
|
|
|
else
|
|
|
|
|
warn "launchd agent not loaded — this script is the only thing that will restart the daemon."
|
|
|
|
|
warn "systemd --user unit NOT installed ($SYSTEMD_UNIT)."
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# fleetd #492: decide which of the two (if either) actually supervises this daemon, and refuse
|
|
|
|
|
# outright — before touching anything — if that cannot be told apart (see require_drivable_
|
|
|
|
|
# supervisor above). --check reaches this same line, so a host with an undrivable supervisor is
|
|
|
|
|
# reported as a failure even in --check, without ever reaching the build/stop/start steps.
|
|
|
|
|
SUPERVISOR_KIND="$(detect_supervisor)"
|
|
|
|
|
require_drivable_supervisor "$SUPERVISOR_KIND"
|
|
|
|
|
ok "supervisor detected: $SUPERVISOR_KIND"
|
|
|
|
|
SUPERVISED=0
|
|
|
|
|
case "$SUPERVISOR_KIND" in
|
|
|
|
|
launchd)
|
|
|
|
|
SUPERVISED=1
|
|
|
|
|
ok "launchd agent loaded ($LAUNCHD_LABEL) — launchd supervises this daemon"
|
|
|
|
|
# CB-600: fail loudly here, before ANY other check runs, if this script and the loaded plist
|
|
|
|
|
# would read different log files — every check after this point is worthless otherwise.
|
|
|
|
|
check_log_path_matches_plist "$OUT" "$LAUNCHD_PLIST"
|
|
|
|
|
;;
|
|
|
|
|
systemd)
|
|
|
|
|
SUPERVISED=1
|
|
|
|
|
ok "systemd --user unit active ($SYSTEMD_UNIT) — systemd supervises this daemon"
|
|
|
|
|
;;
|
|
|
|
|
none)
|
|
|
|
|
warn "no supervisor loaded — this script is the only thing that will restart the daemon."
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
|
|
|
|
|
# The trap with no log line. Checked in a LOGIN shell, because that is how the daemon is started
|
|
|
|
|
# below. Never prints the value — only whether it resolved.
|
|
|
|
|
if zsh -lc '[ -n "${WORKER_GITEA_TOKEN:-}" ]' 2>/dev/null; then
|
|
|
|
@@ -281,25 +402,39 @@ fi
|
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------ stop
|
|
|
|
|
#
|
|
|
|
|
# CB-594: when SUPERVISED, launchd owns the stop — never a raw `kill` here. A bare SIGTERM makes
|
|
|
|
|
# this JVM exit 143 even with its shutdown hook running to completion (verified separately: a
|
|
|
|
|
# throwaway Java process with an equivalent shutdown hook, sent SIGTERM from a login shell that
|
|
|
|
|
# could `wait` on it directly, reported exit code 143 every time — never 0). launchd's
|
|
|
|
|
# KeepAlive.SuccessfulExit=false treats any nonzero exit as a crash and restarts the OLD jar,
|
|
|
|
|
# which would race this script's own restart of the NEW one. `launchctl unload` avoids that race
|
|
|
|
|
# by deregistering the job first, so no KeepAlive is left armed when the process actually stops.
|
|
|
|
|
# CB-594 / fleetd #492: when SUPERVISED, the supervisor owns the stop — never a raw `kill` here. A
|
|
|
|
|
# bare SIGTERM makes this JVM exit 143 even with its shutdown hook running to completion (verified
|
|
|
|
|
# separately: a throwaway Java process with an equivalent shutdown hook, sent SIGTERM from a login
|
|
|
|
|
# shell that could `wait` on it directly, reported exit code 143 every time — never 0). launchd's
|
|
|
|
|
# KeepAlive.SuccessfulExit=false and systemd's Restart=on-failure both treat any nonzero exit as a
|
|
|
|
|
# crash and restart the OLD jar, which would race this script's own restart of the NEW one.
|
|
|
|
|
# `launchctl unload` avoids that race by deregistering the job first, so no KeepAlive is left
|
|
|
|
|
# armed when the process actually stops. `systemctl --user stop` needs no such dance: unlike
|
|
|
|
|
# KeepAlive, systemd's Restart= does not fire on a deliberate stop, only on an unexpected exit of
|
|
|
|
|
# an active unit.
|
|
|
|
|
|
|
|
|
|
if [ -n "$OLD_PID" ]; then
|
|
|
|
|
say "stop"
|
|
|
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)" # verify a FRESH line appears later
|
|
|
|
|
if [ "$SUPERVISED" = 1 ]; then
|
|
|
|
|
echo " supervision is ON: using 'launchctl unload' (not kill) so launchd's own KeepAlive"
|
|
|
|
|
echo " cannot restart the OLD jar out from under this script — see the CB-594 comment above."
|
|
|
|
|
launchctl unload -w "$LAUNCHD_PLIST" \
|
|
|
|
|
|| die "launchctl unload failed — the daemon may still be under supervision; investigate before retrying"
|
|
|
|
|
else
|
|
|
|
|
kill "$OLD_PID"
|
|
|
|
|
fi
|
|
|
|
|
case "$SUPERVISOR_KIND" in
|
|
|
|
|
launchd)
|
|
|
|
|
echo " supervision is ON (launchd): using 'launchctl unload' (not kill) so launchd's own"
|
|
|
|
|
echo " KeepAlive cannot restart the OLD jar out from under this script — see the CB-594"
|
|
|
|
|
echo " comment above."
|
|
|
|
|
launchctl unload -w "$LAUNCHD_PLIST" \
|
|
|
|
|
|| die "launchctl unload failed — the daemon may still be under supervision; investigate before retrying"
|
|
|
|
|
;;
|
|
|
|
|
systemd)
|
|
|
|
|
echo " supervision is ON (systemd --user): using 'systemctl --user stop' (not kill) so"
|
|
|
|
|
echo " systemd's own Restart=on-failure cannot restart the OLD jar out from under this"
|
|
|
|
|
echo " script — see the fleetd #492 comment above."
|
|
|
|
|
systemctl --user stop "$SYSTEMD_UNIT" \
|
|
|
|
|
|| die "'systemctl --user stop $SYSTEMD_UNIT' failed — the daemon may still be under supervision; investigate before retrying"
|
|
|
|
|
;;
|
|
|
|
|
none)
|
|
|
|
|
kill "$OLD_PID"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
for _ in $(seq "$STOP_WAIT"); do
|
|
|
|
|
[ -z "$(running_pid)" ] && break
|
|
|
|
|
sleep 1
|
|
|
|
@@ -310,13 +445,20 @@ if [ -n "$OLD_PID" ]; then
|
|
|
|
|
leave worktrees and panes behind. Investigate, then kill -9 by hand if you accept that."
|
|
|
|
|
fi
|
|
|
|
|
ok "pid $OLD_PID exited"
|
|
|
|
|
elif [ "$SUPERVISED" = 1 ]; then
|
|
|
|
|
elif [ "$SUPERVISOR_KIND" = "launchd" ]; then
|
|
|
|
|
# Loaded but not currently running (e.g. throttled after a crash loop). Unload it anyway so the
|
|
|
|
|
# start step below does a clean load, never a load stacked on an already-loaded label.
|
|
|
|
|
say "stop"
|
|
|
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)"
|
|
|
|
|
launchctl unload -w "$LAUNCHD_PLIST" 2>/dev/null || true
|
|
|
|
|
ok "launchd agent unloaded (was already not running)"
|
|
|
|
|
elif [ "$SUPERVISOR_KIND" = "systemd" ]; then
|
|
|
|
|
# Same case for systemd: the unit is known/active-capable but not currently running. `stop` on an
|
|
|
|
|
# already-stopped unit is a harmless no-op — kept for symmetry with the launchd branch above.
|
|
|
|
|
say "stop"
|
|
|
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)"
|
|
|
|
|
systemctl --user stop "$SYSTEMD_UNIT" 2>/dev/null || true
|
|
|
|
|
ok "systemd --user unit stopped (was already not running)"
|
|
|
|
|
else
|
|
|
|
|
RESTART_MARK="$(wc -l < "$OUT" 2>/dev/null || echo 0)"
|
|
|
|
|
fi
|
|
|
|
@@ -324,34 +466,48 @@ fi
|
|
|
|
|
# ------------------------------------------------------------------ start
|
|
|
|
|
# Unsupervised: login shell (zsh -l) is what puts the secrets on the daemon's environment, and cwd
|
|
|
|
|
# must be fleetd/ because the daemon resolves fleetd.yaml, logs/ and target/ relative to it.
|
|
|
|
|
# Supervised: launchd does both — deploy/dev.ltms.fleetd.plist points ProgramArguments at
|
|
|
|
|
# Supervised (launchd): launchd does both — deploy/dev.ltms.fleetd.plist points ProgramArguments at
|
|
|
|
|
# scripts/fleetd-launchd-wrapper.sh (CB-594), which is what execs the login shell in launchd's
|
|
|
|
|
# place, and WorkingDirectory in the plist already pins fleetd/.
|
|
|
|
|
# Supervised (systemd --user): the unit does both too — measured on the second host, ExecStart is
|
|
|
|
|
# `/bin/zsh -lc "exec java -jar target/fleetd.jar fleetd.yaml"` (a login shell, same reason as
|
|
|
|
|
# above) and WorkingDirectory is already pinned to fleetd/.
|
|
|
|
|
|
|
|
|
|
say "start"
|
|
|
|
|
if [ "$SUPERVISED" = 1 ]; then
|
|
|
|
|
echo " supervision is ON: using 'launchctl load' so launchd starts and keeps supervising this"
|
|
|
|
|
echo " process, instead of a manual nohup that launchd would know nothing about."
|
|
|
|
|
# CB-600: 'launchctl unload -w' above already persisted Disabled=true for this label. A load -w
|
|
|
|
|
# that succeeds clears it; a load -w that FAILS leaves the agent both stopped and disabled — worse
|
|
|
|
|
# than before this script ran, because a later reboot or login will not bring it back either. One
|
|
|
|
|
# retry covers a transient race (e.g. launchd not yet fully done deregistering); if it still fails,
|
|
|
|
|
# die with the exact recovery command rather than a bare "failed".
|
|
|
|
|
if ! launchctl load -w "$LAUNCHD_PLIST" 2>/dev/null; then
|
|
|
|
|
warn "launchctl load failed on the first attempt — retrying once after a short pause"
|
|
|
|
|
sleep 2
|
|
|
|
|
launchctl load -w "$LAUNCHD_PLIST" || die "launchctl load failed twice.
|
|
|
|
|
The agent is now STOPPED and DISABLED — it will NOT come back on its own, not even after a
|
|
|
|
|
reboot or login, because 'launchctl unload -w' above persisted Disabled=true and load -w
|
|
|
|
|
never got the chance to clear it. Recover with:
|
|
|
|
|
launchctl load -w \"$LAUNCHD_PLIST\"
|
|
|
|
|
If that still fails, check 'launchctl list $LAUNCHD_LABEL', validate the plist with
|
|
|
|
|
'plutil -lint \"$LAUNCHD_PLIST\"', and check $OUT before assuming a retry will succeed."
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
# Absolute jar path so `ps` names which checkout is running.
|
|
|
|
|
( cd "$MODULE" && zsh -lc "nohup java -jar '$JAR' >> fleetd.out 2>&1 &" )
|
|
|
|
|
fi
|
|
|
|
|
case "$SUPERVISOR_KIND" in
|
|
|
|
|
launchd)
|
|
|
|
|
echo " supervision is ON (launchd): using 'launchctl load' so launchd starts and keeps"
|
|
|
|
|
echo " supervising this process, instead of a manual nohup that launchd would know nothing"
|
|
|
|
|
echo " about."
|
|
|
|
|
# CB-600: 'launchctl unload -w' above already persisted Disabled=true for this label. A load -w
|
|
|
|
|
# that succeeds clears it; a load -w that FAILS leaves the agent both stopped and disabled — worse
|
|
|
|
|
# than before this script ran, because a later reboot or login will not bring it back either. One
|
|
|
|
|
# retry covers a transient race (e.g. launchd not yet fully done deregistering); if it still fails,
|
|
|
|
|
# die with the exact recovery command rather than a bare "failed".
|
|
|
|
|
if ! launchctl load -w "$LAUNCHD_PLIST" 2>/dev/null; then
|
|
|
|
|
warn "launchctl load failed on the first attempt — retrying once after a short pause"
|
|
|
|
|
sleep 2
|
|
|
|
|
launchctl load -w "$LAUNCHD_PLIST" || die "launchctl load failed twice.
|
|
|
|
|
The agent is now STOPPED and DISABLED — it will NOT come back on its own, not even after a
|
|
|
|
|
reboot or login, because 'launchctl unload -w' above persisted Disabled=true and load -w
|
|
|
|
|
never got the chance to clear it. Recover with:
|
|
|
|
|
launchctl load -w \"$LAUNCHD_PLIST\"
|
|
|
|
|
If that still fails, check 'launchctl list $LAUNCHD_LABEL', validate the plist with
|
|
|
|
|
'plutil -lint \"$LAUNCHD_PLIST\"', and check $OUT before assuming a retry will succeed."
|
|
|
|
|
fi
|
|
|
|
|
;;
|
|
|
|
|
systemd)
|
|
|
|
|
echo " supervision is ON (systemd --user): using 'systemctl --user start' so systemd starts"
|
|
|
|
|
echo " and keeps supervising this process, instead of a manual nohup it would know nothing"
|
|
|
|
|
echo " about."
|
|
|
|
|
systemctl --user start "$SYSTEMD_UNIT" || die "'systemctl --user start $SYSTEMD_UNIT' failed.
|
|
|
|
|
Check 'systemctl --user status $SYSTEMD_UNIT' and $OUT before assuming a retry will succeed."
|
|
|
|
|
;;
|
|
|
|
|
none)
|
|
|
|
|
# Absolute jar path so `ps` names which checkout is running.
|
|
|
|
|
( cd "$MODULE" && zsh -lc "nohup java -jar '$JAR' >> fleetd.out 2>&1 &" )
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
|
|
|
|
|
for _ in $(seq 10); do
|
|
|
|
|
NEW_PID="$(running_pid)"
|
|
|
|
@@ -411,6 +567,13 @@ FRESH_LOG="$(mktemp -t fleetd-fresh-log)"
|
|
|
|
|
trap 'rm -f "$FRESH_LOG"' EXIT
|
|
|
|
|
tail -n "+$((RESTART_MARK + 1))" "$OUT" > "$FRESH_LOG" 2>/dev/null || true
|
|
|
|
|
classify_amqp_connection_errors "$FRESH_LOG"
|
|
|
|
|
|
|
|
|
|
# fleetd #492: checked here, after healthz and the fresh-log check have both had time to run, so a
|
|
|
|
|
# supervisor that revives the OLD jar a few seconds late is caught too. Every check above (healthz
|
|
|
|
|
# 200, jar id, the fresh 'listening' line) is satisfied by EITHER daemon if two are alive — this is
|
|
|
|
|
# the only one that can tell.
|
|
|
|
|
assert_single_daemon "$(running_pid)"
|
|
|
|
|
|
|
|
|
|
say "result"
|
|
|
|
|
ok "pid $NEW_PID, jar $(jar_id)"
|
|
|
|
|
if [ "$REDEPLOY_ERROR_COUNT" -eq 0 ]; then
|
|
|
|
|
Dai Ha
ltms