fleetd/scripts/test-redeploy-fleetd.sh

#!/usr/bin/env bash
# Self-contained checks for the pure log classifier in redeploy-fleetd.sh.

set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TMP="$(mktemp -d "$ROOT/.redeploy-log-test.XXXXXX")"
trap 'rm -rf "$TMP"' EXIT

# Sourcing stops before redeploy-fleetd.sh can build, stop, or start the daemon.
source "$ROOT/scripts/redeploy-fleetd.sh"

fail() {
  printf 'FAIL: %s\n' "$*" >&2
  return 1
}

assert_equals() {
  local expected="$1" actual="$2" description="$3"
  [ "$expected" = "$actual" ] || fail "$description: expected $expected, got $actual"
}

classify_fixture() {
  local name="$1"
  classify_amqp_connection_errors "$TMP/$name"
}

test_no_errors() {
  cat > "$TMP/no-errors.log" <<'LOG'
2026-09-05 12:00:00 INFO fleetd listening
LOG
  classify_fixture no-errors.log
  assert_equals 0 "$REDEPLOY_ERROR_COUNT" "no-errors total"
  assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "no-errors unexplained"
}

test_recovery_patterns_match_source() {
  grep -F 'AMQP connection recovered; cleared held replies for fresh redelivery' \
    "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/AmqpReplyInbox.java" > /dev/null \
    || fail "reply-inbox recovery pattern no longer matches source"
  grep -F 'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery' \
    "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/LeadMailbox.java" > /dev/null \
    || fail "lead-mailbox recovery pattern no longer matches source"
}

test_attributed_recovered_connection_error() {
  cat > "$TMP/attributed-recovered.log" <<'LOG'
2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred
2026-09-05 12:00:01 INFO AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
LOG
  classify_fixture attributed-recovered.log
  assert_equals 1 "$REDEPLOY_ERROR_COUNT" "attributed-recovered total"
  assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "attributed-recovered errors"
  assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "attributed-recovered unexplained"
}

test_real_error_shape_is_loud_when_unattributable() {
  # The real handler ERROR lines name neither inbox nor lead mailbox. Even though both connections
  # later recover, this region must stay loud because the recovery cannot be assigned safely.
  cat > "$TMP/real-error-shape.log" <<'LOG'
17:37:53.537 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred
18:20:33.027 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery!
18:30:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred
19:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery!
20:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred
21:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery!
21:41:39.329 INFO  [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
21:41:48.693 INFO  [AMQP Connection 10.10.20.13:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
LOG
  classify_fixture real-error-shape.log
  assert_equals 6 "$REDEPLOY_ERROR_COUNT" "real-error-shape total"
  assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "real-error-shape recovered"
  assert_equals 6 "$REDEPLOY_UNEXPLAINED_ERRORS" "real-error-shape unexplained"
}

test_cross_connection_unattributable_errors_stay_loud() {
  # This is the reported unsafe shape. Both handler lines lack a connection kind, so LeadMailbox
  # recoveries must not consume either one.
  cat > "$TMP/cross-unattributable.log" <<'LOG'
2026-09-05 12:00:00 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred
2026-09-05 12:00:01 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred
2026-09-05 12:00:02 INFO  [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
2026-09-05 12:00:03 INFO  [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
LOG
  classify_fixture cross-unattributable.log
  assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-unattributable total"
  assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-unattributable recovered"
  assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-unattributable unexplained"
}

test_attributed_cross_connection_errors_stay_loud() {
  # This fixture exercises the per-kind pending state should a future layout include a simple class
  # name on the handler line. LeadMailbox recovery cannot heal AmqpReplyInbox errors.
  cat > "$TMP/cross-attributed.log" <<'LOG'
2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred
2026-09-05 12:00:01 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred
2026-09-05 12:00:02 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
2026-09-05 12:00:03 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery
LOG
  classify_fixture cross-attributed.log
  assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-attributed total"
  assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-attributed recovered"
  assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-attributed unexplained"
}

test_attributed_unrecovered_connection_error() {
  cat > "$TMP/unrecovered.log" <<'LOG'
2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred
LOG
  classify_fixture unrecovered.log
  assert_equals 1 "$REDEPLOY_ERROR_COUNT" "unrecovered total"
  assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "unrecovered AMQP errors"
  assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "unrecovered unexplained"
}

test_other_error_is_unexplained() {
  cat > "$TMP/other-error.log" <<'LOG'
2026-09-05 12:00:00 ERROR dev.ltms.fleet.Fleetd - startup failed
2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery
LOG
  classify_fixture other-error.log
  assert_equals 1 "$REDEPLOY_ERROR_COUNT" "other-error total"
  assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "other-error unexplained"
}

test_recovery_requirement_mutation_is_caught() {
  classify_amqp_connection_errors() {
    local log_file="$1" line
    REDEPLOY_ERROR_COUNT=0
    REDEPLOY_RECOVERED_AMQP_ERRORS=0
    REDEPLOY_UNEXPLAINED_ERRORS=0
    while IFS= read -r line || [ -n "$line" ]; do
      case "$line" in
        *' ERROR '*|*' SEVERE '*)
          REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
          case "$line" in
            *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*)
              REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
              ;;
            *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
          esac
          ;;
      esac
    done < "$log_file"
  }

  if test_attributed_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then
    fail "mutation accepted an unrecovered connection error"
  fi
  grep -F 'FAIL: unrecovered AMQP errors: expected 0, got 1' "$TMP/mutation-output" > /dev/null \
    || fail "mutation failed without the expected assertion"
  printf 'Recovery mutation: FAIL: unrecovered AMQP errors: expected 0, got 1\n'
}

test_shared_counter_mutation_is_caught() {
  classify_amqp_connection_errors() {
    local log_file="$1" line pending=0
    REDEPLOY_ERROR_COUNT=0
    REDEPLOY_RECOVERED_AMQP_ERRORS=0
    REDEPLOY_UNEXPLAINED_ERRORS=0
    while IFS= read -r line || [ -n "$line" ]; do
      case "$line" in
        *' ERROR '*|*' SEVERE '*)
          REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
          case "$line" in
            *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*)
              case "$line" in
                *'AmqpReplyInbox'*|*'LeadMailbox'*) pending=$((pending + 1)) ;;
                *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
              esac
              ;;
            *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
          esac
          ;;
        *'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*)
          if [ "$pending" -gt 0 ]; then
            pending=$((pending - 1))
            REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
          fi
          ;;
      esac
    done < "$log_file"
    REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending))
  }

  if test_attributed_cross_connection_errors_stay_loud > "$TMP/shared-mutation-output" 2>&1; then
    fail "shared counter mutation accepted cross-connection recovery"
  fi
  grep -F 'FAIL: cross-attributed recovered: expected 0, got 2' "$TMP/shared-mutation-output" > /dev/null \
    || fail "shared counter mutation failed without the expected assertion"
  printf 'Shared-counter mutation: FAIL: cross-attributed recovered: expected 0, got 2\n'
}

test_no_errors
test_recovery_patterns_match_source
test_attributed_recovered_connection_error
test_real_error_shape_is_loud_when_unattributable
test_cross_connection_unattributable_errors_stay_loud
test_attributed_cross_connection_errors_stay_loud
test_attributed_unrecovered_connection_error
test_other_error_is_unexplained
test_recovery_requirement_mutation_is_caught
test_shared_counter_mutation_is_caught
printf 'PASS: redeploy log classifier\n'