#!/usr/bin/env bash # Self-contained checks for the pure log classifier in redeploy-fleetd.sh. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" TMP="$(mktemp -d "$ROOT/.redeploy-log-test.XXXXXX")" trap 'rm -rf "$TMP"' EXIT # Sourcing stops before redeploy-fleetd.sh can build, stop, or start the daemon. source "$ROOT/scripts/redeploy-fleetd.sh" fail() { printf 'FAIL: %s\n' "$*" >&2 return 1 } assert_equals() { local expected="$1" actual="$2" description="$3" [ "$expected" = "$actual" ] || fail "$description: expected $expected, got $actual" } classify_fixture() { local name="$1" classify_amqp_connection_errors "$TMP/$name" } test_no_errors() { cat > "$TMP/no-errors.log" <<'LOG' 2026-09-05 12:00:00 INFO fleetd listening LOG classify_fixture no-errors.log assert_equals 0 "$REDEPLOY_ERROR_COUNT" "no-errors total" assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "no-errors unexplained" } test_recovery_patterns_match_source() { grep -F 'AMQP connection recovered; cleared held replies for fresh redelivery' \ "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/AmqpReplyInbox.java" > /dev/null \ || fail "reply-inbox recovery pattern no longer matches source" grep -F 'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery' \ "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/LeadMailbox.java" > /dev/null \ || fail "lead-mailbox recovery pattern no longer matches source" } test_attributed_recovered_connection_error() { cat > "$TMP/attributed-recovered.log" <<'LOG' 2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred 2026-09-05 12:00:01 INFO AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery LOG classify_fixture attributed-recovered.log assert_equals 1 "$REDEPLOY_ERROR_COUNT" "attributed-recovered total" assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "attributed-recovered errors" assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "attributed-recovered unexplained" } test_real_error_shape_is_loud_when_unattributable() { # The real handler ERROR lines name neither inbox nor lead mailbox. Even though both connections # later recover, this region must stay loud because the recovery cannot be assigned safely. cat > "$TMP/real-error-shape.log" <<'LOG' 17:37:53.537 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred 18:20:33.027 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! 18:30:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred 19:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! 20:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred 21:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! 21:41:39.329 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery 21:41:48.693 INFO [AMQP Connection 10.10.20.13:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery LOG classify_fixture real-error-shape.log assert_equals 6 "$REDEPLOY_ERROR_COUNT" "real-error-shape total" assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "real-error-shape recovered" assert_equals 6 "$REDEPLOY_UNEXPLAINED_ERRORS" "real-error-shape unexplained" } test_cross_connection_unattributable_errors_stay_loud() { # This is the reported unsafe shape. Both handler lines lack a connection kind, so LeadMailbox # recoveries must not consume either one. cat > "$TMP/cross-unattributable.log" <<'LOG' 2026-09-05 12:00:00 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred 2026-09-05 12:00:01 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred 2026-09-05 12:00:02 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery 2026-09-05 12:00:03 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery LOG classify_fixture cross-unattributable.log assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-unattributable total" assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-unattributable recovered" assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-unattributable unexplained" } test_attributed_cross_connection_errors_stay_loud() { # This fixture exercises the per-kind pending state should a future layout include a simple class # name on the handler line. LeadMailbox recovery cannot heal AmqpReplyInbox errors. cat > "$TMP/cross-attributed.log" <<'LOG' 2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred 2026-09-05 12:00:01 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred 2026-09-05 12:00:02 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery 2026-09-05 12:00:03 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery LOG classify_fixture cross-attributed.log assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-attributed total" assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-attributed recovered" assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-attributed unexplained" } test_attributed_unrecovered_connection_error() { cat > "$TMP/unrecovered.log" <<'LOG' 2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred LOG classify_fixture unrecovered.log assert_equals 1 "$REDEPLOY_ERROR_COUNT" "unrecovered total" assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "unrecovered AMQP errors" assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "unrecovered unexplained" } test_other_error_is_unexplained() { cat > "$TMP/other-error.log" <<'LOG' 2026-09-05 12:00:00 ERROR dev.ltms.fleet.Fleetd - startup failed 2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery LOG classify_fixture other-error.log assert_equals 1 "$REDEPLOY_ERROR_COUNT" "other-error total" assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "other-error unexplained" } test_recovery_requirement_mutation_is_caught() { classify_amqp_connection_errors() { local log_file="$1" line REDEPLOY_ERROR_COUNT=0 REDEPLOY_RECOVERED_AMQP_ERRORS=0 REDEPLOY_UNEXPLAINED_ERRORS=0 while IFS= read -r line || [ -n "$line" ]; do case "$line" in *' ERROR '*|*' SEVERE '*) REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) case "$line" in *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*) REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) ;; *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; esac ;; esac done < "$log_file" } if test_attributed_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then fail "mutation accepted an unrecovered connection error" fi grep -F 'FAIL: unrecovered AMQP errors: expected 0, got 1' "$TMP/mutation-output" > /dev/null \ || fail "mutation failed without the expected assertion" printf 'Recovery mutation: FAIL: unrecovered AMQP errors: expected 0, got 1\n' } test_shared_counter_mutation_is_caught() { classify_amqp_connection_errors() { local log_file="$1" line pending=0 REDEPLOY_ERROR_COUNT=0 REDEPLOY_RECOVERED_AMQP_ERRORS=0 REDEPLOY_UNEXPLAINED_ERRORS=0 while IFS= read -r line || [ -n "$line" ]; do case "$line" in *' ERROR '*|*' SEVERE '*) REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) case "$line" in *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*) case "$line" in *'AmqpReplyInbox'*|*'LeadMailbox'*) pending=$((pending + 1)) ;; *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; esac ;; *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; esac ;; *'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*) if [ "$pending" -gt 0 ]; then pending=$((pending - 1)) REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) fi ;; esac done < "$log_file" REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending)) } if test_attributed_cross_connection_errors_stay_loud > "$TMP/shared-mutation-output" 2>&1; then fail "shared counter mutation accepted cross-connection recovery" fi grep -F 'FAIL: cross-attributed recovered: expected 0, got 2' "$TMP/shared-mutation-output" > /dev/null \ || fail "shared counter mutation failed without the expected assertion" printf 'Shared-counter mutation: FAIL: cross-attributed recovered: expected 0, got 2\n' } test_no_errors test_recovery_patterns_match_source test_attributed_recovered_connection_error test_real_error_shape_is_loud_when_unattributable test_cross_connection_unattributable_errors_stay_loud test_attributed_cross_connection_errors_stay_loud test_attributed_unrecovered_connection_error test_other_error_is_unexplained test_recovery_requirement_mutation_is_caught test_shared_counter_mutation_is_caught printf 'PASS: redeploy log classifier\n'