Dai Ha
dcd505286f
launchd, systemd --user, and unsupervised are three different answers, not two. Refuse (die) rather than fall through to kill+nohup when a supervisor is detected that this script cannot drive (e.g. both signals fire at once), and add a post-restart check that fails the run if more than one fleetd process is alive. detect_supervisor()/require_drivable_supervisor()/ count_daemon_pids()/assert_single_daemon() are pure, overridable functions so scripts/test-redeploy-fleetd.sh can exercise them without a real launchd or systemd.
319 lines
16 KiB
Bash
Executable File
319 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Self-contained checks for the pure log classifier in redeploy-fleetd.sh.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
TMP="$(mktemp -d "$ROOT/.redeploy-log-test.XXXXXX")"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
|
|
# Sourcing stops before redeploy-fleetd.sh can build, stop, or start the daemon.
|
|
source "$ROOT/scripts/redeploy-fleetd.sh"
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$*" >&2
|
|
return 1
|
|
}
|
|
|
|
assert_equals() {
|
|
local expected="$1" actual="$2" description="$3"
|
|
[ "$expected" = "$actual" ] || fail "$description: expected $expected, got $actual"
|
|
}
|
|
|
|
classify_fixture() {
|
|
local name="$1"
|
|
classify_amqp_connection_errors "$TMP/$name"
|
|
}
|
|
|
|
|
|
# fleetd #492 — supervisor detection. Detect_supervisor() reads launchd_loaded/systemd_loaded, so
|
|
# each test overrides BOTH pairs (installed + loaded) explicitly, rather than relying on either
|
|
# being naturally absent: this machine may itself be running a real fleetd under launchd right now
|
|
# (see CLAUDE.md/MEMORY.md — launchd supervision has been live here since 2026-08-26), so leaving
|
|
# launchd_loaded unmocked in a "systemd only" test would silently read this host's own live state
|
|
# instead of the fixture.
|
|
test_detect_supervisor_launchd_only() {
|
|
launchd_installed() { return 0; }
|
|
launchd_loaded() { return 0; }
|
|
systemd_installed() { return 1; }
|
|
systemd_loaded() { return 1; }
|
|
assert_equals "launchd" "$(detect_supervisor)" "launchd-only detection"
|
|
}
|
|
|
|
test_detect_supervisor_systemd_only() {
|
|
launchd_installed() { return 1; }
|
|
launchd_loaded() { return 1; }
|
|
systemd_installed() { return 0; }
|
|
systemd_loaded() { return 0; }
|
|
assert_equals "systemd" "$(detect_supervisor)" "systemd-only detection"
|
|
}
|
|
|
|
test_detect_supervisor_none() {
|
|
launchd_installed() { return 1; }
|
|
launchd_loaded() { return 1; }
|
|
systemd_installed() { return 1; }
|
|
systemd_loaded() { return 1; }
|
|
assert_equals "none" "$(detect_supervisor)" "unsupervised detection"
|
|
}
|
|
|
|
# The heart of the ticket: a supervisor this script cannot drive must refuse, never fall through to
|
|
# `kill`. require_drivable_supervisor die()s, so it is invoked inside a command substitution — that
|
|
# forks a subshell, so its exit() only ends the subshell and this test script keeps running under
|
|
# `set -e`.
|
|
test_require_drivable_supervisor_refuses_ambiguous() {
|
|
local output rc=0
|
|
output="$(require_drivable_supervisor "ambiguous" 2>&1)" || rc=$?
|
|
[ "$rc" -ne 0 ] || fail "require_drivable_supervisor accepted an ambiguous (undrivable) supervisor"
|
|
printf '%s' "$output" | grep -qF "$LAUNCHD_LABEL" \
|
|
|| fail "refusal message does not name the launchd label it found"
|
|
printf '%s' "$output" | grep -qF "$SYSTEMD_UNIT" \
|
|
|| fail "refusal message does not name the systemd unit it found"
|
|
}
|
|
|
|
test_require_drivable_supervisor_accepts_known_kinds() {
|
|
require_drivable_supervisor "launchd" || fail "refused a drivable launchd supervisor"
|
|
require_drivable_supervisor "systemd" || fail "refused a drivable systemd supervisor"
|
|
require_drivable_supervisor "none" || fail "refused the unsupervised case"
|
|
}
|
|
|
|
# fleetd #492 — the one-daemon check. Two live pids is the exact symptom a racing supervisor
|
|
# produces, and none of the other post-restart checks (healthz, jar id, the fresh log line) can see
|
|
# it because either daemon alone satisfies them.
|
|
test_count_daemon_pids() {
|
|
assert_equals 0 "$(count_daemon_pids "")" "count of an empty pid list"
|
|
assert_equals 1 "$(count_daemon_pids "4242")" "count of a single pid"
|
|
assert_equals 2 "$(count_daemon_pids "$(printf '4242\n4343\n')")" "count of two pids"
|
|
}
|
|
|
|
test_assert_single_daemon_accepts_one_pid() {
|
|
assert_single_daemon "4242" || fail "assert_single_daemon rejected a single running pid"
|
|
}
|
|
|
|
test_assert_single_daemon_rejects_two_pids() {
|
|
local output rc=0
|
|
output="$(assert_single_daemon "$(printf '4242\n4343\n')" 2>&1)" || rc=$?
|
|
[ "$rc" -ne 0 ] || fail "assert_single_daemon accepted two simultaneously running pids"
|
|
printf '%s' "$output" | grep -qF '4242' || fail "refusal message does not list the pids it found"
|
|
printf '%s' "$output" | grep -qF '4343' || fail "refusal message does not list the pids it found"
|
|
}
|
|
|
|
test_no_errors() {
|
|
cat > "$TMP/no-errors.log" <<'LOG'
|
|
2026-09-05 12:00:00 INFO fleetd listening
|
|
LOG
|
|
classify_fixture no-errors.log
|
|
assert_equals 0 "$REDEPLOY_ERROR_COUNT" "no-errors total"
|
|
assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "no-errors unexplained"
|
|
}
|
|
|
|
test_recovery_patterns_match_source() {
|
|
grep -F 'AMQP connection {}: {}' "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/AmqpReplyInbox.java" > /dev/null \
|
|
|| fail "AMQP failure pattern no longer matches source"
|
|
grep -F 'AMQP connection recovered; cleared held replies for fresh redelivery' \
|
|
"$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/AmqpReplyInbox.java" > /dev/null \
|
|
|| fail "reply-inbox recovery pattern no longer matches source"
|
|
grep -F 'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery' \
|
|
"$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/LeadMailbox.java" > /dev/null \
|
|
|| fail "lead-mailbox recovery pattern no longer matches source"
|
|
}
|
|
|
|
test_attributed_recovered_connection_error() {
|
|
cat > "$TMP/attributed-recovered.log" <<'LOG'
|
|
2026-09-05 12:00:00 ERROR [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
2026-09-05 12:00:01 INFO [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
|
|
LOG
|
|
classify_fixture attributed-recovered.log
|
|
assert_equals 1 "$REDEPLOY_ERROR_COUNT" "attributed-recovered total"
|
|
assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "attributed-recovered errors"
|
|
assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "attributed-recovered unexplained"
|
|
}
|
|
|
|
test_source_derived_error_shapes_recover_by_connection() {
|
|
# These ERROR shapes come from AmqpConnectionFailureLogger on main. They need a live-log check
|
|
# after redeploy because the new code has not yet written a production line.
|
|
cat > "$TMP/source-derived.log" <<'LOG'
|
|
17:37:53.537 ERROR [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
17:37:54.537 ERROR [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection fleetd-reply-inbox: Caught an exception during connection recovery!
|
|
17:37:55.537 ERROR [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
17:37:56.537 ERROR [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP connection fleetd-lead-mailbox: An unexpected connection driver error occurred
|
|
17:37:57.537 ERROR [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP connection fleetd-lead-mailbox: Caught an exception during connection recovery!
|
|
17:37:58.537 ERROR [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP connection fleetd-lead-mailbox: An unexpected connection driver error occurred
|
|
17:38:00.000 INFO [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
|
|
17:38:01.000 INFO [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
|
|
17:38:02.000 INFO [AMQP Connection broker:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
|
|
17:38:03.000 INFO [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
17:38:04.000 INFO [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
17:38:05.000 INFO [AMQP Connection broker:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
LOG
|
|
classify_fixture source-derived.log
|
|
assert_equals 6 "$REDEPLOY_ERROR_COUNT" "source-derived total"
|
|
assert_equals 6 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "source-derived recovered"
|
|
assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "source-derived unexplained"
|
|
}
|
|
|
|
test_cross_connection_unattributable_errors_stay_loud() {
|
|
# This candidate has neither stable connection name, so LeadMailbox recovery must not consume it.
|
|
cat > "$TMP/cross-unattributable.log" <<'LOG'
|
|
2026-09-05 12:00:00 ERROR [AMQP Connection broker:5672] unknown - AMQP connection: An unexpected connection driver error occurred
|
|
2026-09-05 12:00:01 ERROR [AMQP Connection broker:5672] unknown - AMQP connection: An unexpected connection driver error occurred
|
|
2026-09-05 12:00:02 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
2026-09-05 12:00:03 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
LOG
|
|
classify_fixture cross-unattributable.log
|
|
assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-unattributable total"
|
|
assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-unattributable recovered"
|
|
assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-unattributable unexplained"
|
|
}
|
|
|
|
test_attributed_cross_connection_errors_stay_loud() {
|
|
# LeadMailbox recovery cannot heal AmqpReplyInbox errors.
|
|
cat > "$TMP/cross-attributed.log" <<'LOG'
|
|
2026-09-05 12:00:00 ERROR AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
2026-09-05 12:00:01 ERROR AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
2026-09-05 12:00:02 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
2026-09-05 12:00:03 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
|
|
LOG
|
|
classify_fixture cross-attributed.log
|
|
assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-attributed total"
|
|
assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-attributed recovered"
|
|
assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-attributed unexplained"
|
|
}
|
|
|
|
test_attributed_unrecovered_connection_error() {
|
|
cat > "$TMP/unrecovered.log" <<'LOG'
|
|
2026-09-05 12:00:00 ERROR AmqpReplyInbox - AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred
|
|
LOG
|
|
classify_fixture unrecovered.log
|
|
assert_equals 1 "$REDEPLOY_ERROR_COUNT" "unrecovered total"
|
|
assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "unrecovered AMQP errors"
|
|
assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "unrecovered unexplained"
|
|
}
|
|
|
|
test_other_error_is_unexplained() {
|
|
cat > "$TMP/other-error.log" <<'LOG'
|
|
2026-09-05 12:00:00 ERROR dev.ltms.fleet.Fleetd - startup failed
|
|
2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
|
|
LOG
|
|
classify_fixture other-error.log
|
|
assert_equals 1 "$REDEPLOY_ERROR_COUNT" "other-error total"
|
|
assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "other-error unexplained"
|
|
}
|
|
|
|
test_recovery_requirement_mutation_is_caught() {
|
|
classify_amqp_connection_errors() {
|
|
local log_file="$1" line
|
|
REDEPLOY_ERROR_COUNT=0
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=0
|
|
REDEPLOY_UNEXPLAINED_ERRORS=0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
*' ERROR '*|*' SEVERE '*)
|
|
REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
|
|
case "$line" in
|
|
*'AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred'*)
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
|
|
;;
|
|
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < "$log_file"
|
|
}
|
|
|
|
if test_attributed_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then
|
|
fail "mutation accepted an unrecovered connection error"
|
|
fi
|
|
grep -F 'FAIL: unrecovered AMQP errors: expected 0, got 1' "$TMP/mutation-output" > /dev/null \
|
|
|| fail "mutation failed without the expected assertion"
|
|
printf 'Recovery mutation: FAIL: unrecovered AMQP errors: expected 0, got 1\n'
|
|
}
|
|
|
|
test_shared_counter_mutation_is_caught() {
|
|
classify_amqp_connection_errors() {
|
|
local log_file="$1" line pending=0
|
|
REDEPLOY_ERROR_COUNT=0
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=0
|
|
REDEPLOY_UNEXPLAINED_ERRORS=0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
*' ERROR '*|*' SEVERE '*)
|
|
REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
|
|
case "$line" in
|
|
*'AMQP connection'*'An unexpected connection driver error occurred'*|*'AMQP connection'*'Caught an exception during connection recovery!'*)
|
|
case "$line" in
|
|
*'fleetd-reply-inbox'*|*'fleetd-lead-mailbox'*) pending=$((pending + 1)) ;;
|
|
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
|
|
esac
|
|
;;
|
|
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
|
|
esac
|
|
;;
|
|
*'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*)
|
|
if [ "$pending" -gt 0 ]; then
|
|
pending=$((pending - 1))
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
|
|
fi
|
|
;;
|
|
esac
|
|
done < "$log_file"
|
|
REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending))
|
|
}
|
|
|
|
if test_attributed_cross_connection_errors_stay_loud > "$TMP/shared-mutation-output" 2>&1; then
|
|
fail "shared counter mutation accepted cross-connection recovery"
|
|
fi
|
|
grep -F 'FAIL: cross-attributed recovered: expected 0, got 2' "$TMP/shared-mutation-output" > /dev/null \
|
|
|| fail "shared counter mutation failed without the expected assertion"
|
|
printf 'Shared-counter mutation: FAIL: cross-attributed recovered: expected 0, got 2\n'
|
|
}
|
|
|
|
test_unattributable_quiet_mutation_is_caught() {
|
|
classify_amqp_connection_errors() {
|
|
local log_file="$1" line
|
|
REDEPLOY_ERROR_COUNT=0
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=0
|
|
REDEPLOY_UNEXPLAINED_ERRORS=0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
*' ERROR '*|*' SEVERE '*)
|
|
REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
|
|
case "$line" in
|
|
*'AMQP connection'*'An unexpected connection driver error occurred'*|*'AMQP connection'*'Caught an exception during connection recovery!'*)
|
|
REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
|
|
;;
|
|
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
|
|
esac
|
|
;;
|
|
esac
|
|
done < "$log_file"
|
|
}
|
|
|
|
if test_cross_connection_unattributable_errors_stay_loud > "$TMP/unattributable-mutation-output" 2>&1; then
|
|
fail "unattributable mutation accepted an unknown connection"
|
|
fi
|
|
grep -F 'FAIL: cross-unattributable recovered: expected 0, got 2' "$TMP/unattributable-mutation-output" > /dev/null \
|
|
|| fail "unattributable mutation failed without the expected assertion"
|
|
printf 'Unattributable mutation: FAIL: cross-unattributable recovered: expected 0, got 2\n'
|
|
}
|
|
|
|
test_detect_supervisor_launchd_only
|
|
test_detect_supervisor_systemd_only
|
|
test_detect_supervisor_none
|
|
test_require_drivable_supervisor_refuses_ambiguous
|
|
test_require_drivable_supervisor_accepts_known_kinds
|
|
test_count_daemon_pids
|
|
test_assert_single_daemon_accepts_one_pid
|
|
test_assert_single_daemon_rejects_two_pids
|
|
test_no_errors
|
|
test_recovery_patterns_match_source
|
|
test_attributed_recovered_connection_error
|
|
test_source_derived_error_shapes_recover_by_connection
|
|
test_cross_connection_unattributable_errors_stay_loud
|
|
test_attributed_cross_connection_errors_stay_loud
|
|
test_attributed_unrecovered_connection_error
|
|
test_other_error_is_unexplained
|
|
test_recovery_requirement_mutation_is_caught
|
|
test_shared_counter_mutation_is_caught
|
|
test_unattributable_quiet_mutation_is_caught
|
|
printf 'PASS: redeploy log classifier\n'
|