diff --git a/scripts/redeploy-fleetd.sh b/scripts/redeploy-fleetd.sh index a2ee281..f6322ab 100755 --- a/scripts/redeploy-fleetd.sh +++ b/scripts/redeploy-fleetd.sh @@ -116,6 +116,41 @@ check_log_path_matches_plist() { ok "log path check: script and plist agree ($resolved_out)" } +# Classify ERROR lines in one fresh log region. RabbitMQ's client reports a lost TCP connection +# through ForgivingExceptionHandler. We only quiet its exact "Connection reset" error after one of +# fleetd's own recovery listeners later reports a completed AMQP recovery. Every other ERROR, and +# every unmatched reset, stays unexplained so a failed broker link remains loud. +classify_amqp_connection_errors() { + local log_file="$1" line pending_resets=0 + REDEPLOY_ERROR_COUNT=0 + REDEPLOY_RECOVERED_AMQP_ERRORS=0 + REDEPLOY_UNEXPLAINED_ERRORS=0 + + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *' ERROR '*|*' SEVERE '*) + REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) + case "$line" in + *'com.rabbitmq.client.impl.ForgivingExceptionHandler'*'Connection reset'*) + pending_resets=$((pending_resets + 1)) + ;; + *) + REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) + ;; + esac + ;; + *'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*) + if [ "$pending_resets" -gt 0 ]; then + pending_resets=$((pending_resets - 1)) + REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) + fi + ;; + esac + done < "$log_file" + + REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending_resets)) +} + # CB-600: sourceable for testing. When this file is SOURCED (not executed) it stops here — nothing # below runs — so a test harness can `source` it to call check_log_path_matches_plist (or the # other pure helpers above) against a throwaway plist fixture without ever reaching the mutating @@ -361,15 +396,21 @@ tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null \ | grep -iE 'deferred|classification:|fleet health:|coverage' | tail -8 | sed 's/^/ /' \ || echo " (nothing reported)" -# Errors since the restart, anchored to the marker so old noise cannot leak in. -ERRS="$(tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null | grep -cE ' (ERROR|SEVERE) ' || true)" +# Errors since the restart, anchored to the marker so old noise cannot leak in. Keep the fresh +# region in a file because the classifier must preserve the order of errors and recoveries. +FRESH_LOG="$(mktemp -t fleetd-fresh-log)" +trap 'rm -f "$FRESH_LOG"' EXIT +tail -n "+$((RESTART_MARK + 1))" "$OUT" > "$FRESH_LOG" 2>/dev/null || true +classify_amqp_connection_errors "$FRESH_LOG" say "result" ok "pid $NEW_PID, jar $(jar_id)" -if [ "${ERRS:-0}" -gt 0 ]; then - warn "$ERRS ERROR lines since restart:" - tail -n "+$((RESTART_MARK + 1))" "$OUT" | grep -E ' (ERROR|SEVERE) ' | tail -5 | sed 's/^/ /' -else +if [ "$REDEPLOY_ERROR_COUNT" -eq 0 ]; then ok "no ERROR lines since restart" +elif [ "$REDEPLOY_UNEXPLAINED_ERRORS" -eq 0 ]; then + ok "$REDEPLOY_RECOVERED_AMQP_ERRORS AMQP connection reset ERROR lines recovered since restart" +else + warn "$REDEPLOY_ERROR_COUNT ERROR lines since restart:" + grep -E ' (ERROR|SEVERE) ' "$FRESH_LOG" | tail -5 | sed 's/^/ /' fi echo echo " Next: call fleet_whoami and confirm it still answers 'primary'. A lead whose tab label" diff --git a/scripts/test-redeploy-fleetd.sh b/scripts/test-redeploy-fleetd.sh new file mode 100755 index 0000000..d16b8e1 --- /dev/null +++ b/scripts/test-redeploy-fleetd.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Self-contained checks for the pure log classifier in redeploy-fleetd.sh. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP="$(mktemp -d "$ROOT/.redeploy-log-test.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +# Sourcing stops before redeploy-fleetd.sh can build, stop, or start the daemon. +source "$ROOT/scripts/redeploy-fleetd.sh" + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + return 1 +} + +assert_equals() { + local expected="$1" actual="$2" description="$3" + [ "$expected" = "$actual" ] || fail "$description: expected $expected, got $actual" +} + +classify_fixture() { + local name="$1" + classify_amqp_connection_errors "$TMP/$name" +} + +test_no_errors() { + cat > "$TMP/no-errors.log" <<'LOG' +2026-09-05 12:00:00 INFO fleetd listening +LOG + classify_fixture no-errors.log + assert_equals 0 "$REDEPLOY_ERROR_COUNT" "no-errors total" + assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "no-errors unexplained" +} + +test_recovered_connection_error() { + cat > "$TMP/recovered.log" <<'LOG' +2026-09-05 12:00:00 ERROR com.rabbitmq.client.impl.ForgivingExceptionHandler - An unexpected connection driver error occurred (Exception message: Connection reset) +2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery +LOG + classify_fixture recovered.log + assert_equals 1 "$REDEPLOY_ERROR_COUNT" "recovered total" + assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "recovered AMQP errors" + assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "recovered unexplained" +} + +test_unrecovered_connection_error() { + cat > "$TMP/unrecovered.log" <<'LOG' +2026-09-05 12:00:00 ERROR com.rabbitmq.client.impl.ForgivingExceptionHandler - An unexpected connection driver error occurred (Exception message: Connection reset) +LOG + classify_fixture unrecovered.log + assert_equals 1 "$REDEPLOY_ERROR_COUNT" "unrecovered total" + assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "unrecovered AMQP errors" + assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "unrecovered unexplained" +} + +test_other_error_is_unexplained() { + cat > "$TMP/other-error.log" <<'LOG' +2026-09-05 12:00:00 ERROR dev.ltms.fleet.Fleetd - startup failed +2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery +LOG + classify_fixture other-error.log + assert_equals 1 "$REDEPLOY_ERROR_COUNT" "other-error total" + assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "other-error unexplained" +} + +test_mutation_is_caught() { + classify_amqp_connection_errors() { + local log_file="$1" line + REDEPLOY_ERROR_COUNT=0 + REDEPLOY_RECOVERED_AMQP_ERRORS=0 + REDEPLOY_UNEXPLAINED_ERRORS=0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *' ERROR '*|*' SEVERE '*) + REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) + case "$line" in + *'com.rabbitmq.client.impl.ForgivingExceptionHandler'*'Connection reset'*) + REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) + ;; + *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; + esac + ;; + esac + done < "$log_file" + } + + if test_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then + fail "mutation accepted an unrecovered connection error" + fi + grep -F 'FAIL: unrecovered AMQP errors: expected 0, got 1' "$TMP/mutation-output" > /dev/null \ + || fail "mutation failed without the expected assertion" + printf 'Mutation check: FAIL: unrecovered AMQP errors: expected 0, got 1\n' +} + +test_no_errors +test_recovered_connection_error +test_unrecovered_connection_error +test_other_error_is_unexplained +test_mutation_is_caught +printf 'PASS: redeploy log classifier\n'