From 0241e0d3a8fefd9ff36e25392f85d5195fc88fb8 Mon Sep 17 00:00:00 2001 From: Dai Ha Date: Sat, 5 Sep 2026 05:36:53 +0700 Subject: [PATCH] Keep unattributed AMQP errors loud --- scripts/redeploy-fleetd.sh | 37 +++++---- scripts/test-redeploy-fleetd.sh | 138 +++++++++++++++++++++++++++----- 2 files changed, 143 insertions(+), 32 deletions(-) diff --git a/scripts/redeploy-fleetd.sh b/scripts/redeploy-fleetd.sh index f6322ab..d2ce73f 100755 --- a/scripts/redeploy-fleetd.sh +++ b/scripts/redeploy-fleetd.sh @@ -116,12 +116,13 @@ check_log_path_matches_plist() { ok "log path check: script and plist agree ($resolved_out)" } -# Classify ERROR lines in one fresh log region. RabbitMQ's client reports a lost TCP connection -# through ForgivingExceptionHandler. We only quiet its exact "Connection reset" error after one of -# fleetd's own recovery listeners later reports a completed AMQP recovery. Every other ERROR, and -# every unmatched reset, stays unexplained so a failed broker link remains loud. +# Classify ERROR lines in one fresh log region. The logging layout abbreviates logger packages, so +# match simple class names and the handler's actual ERROR messages. An error can be quiet only when +# its own line names AmqpReplyInbox or LeadMailbox; the handler lines now seen in fleetd.out name +# neither connection, so they deliberately stay unexplained. This avoids letting one connection's +# recovery hide a failure in the other connection. classify_amqp_connection_errors() { - local log_file="$1" line pending_resets=0 + local log_file="$1" line pending_inbox=0 pending_lead_mailbox=0 REDEPLOY_ERROR_COUNT=0 REDEPLOY_RECOVERED_AMQP_ERRORS=0 REDEPLOY_UNEXPLAINED_ERRORS=0 @@ -131,24 +132,32 @@ classify_amqp_connection_errors() { *' ERROR '*|*' SEVERE '*) REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) case "$line" in - *'com.rabbitmq.client.impl.ForgivingExceptionHandler'*'Connection reset'*) - pending_resets=$((pending_resets + 1)) - ;; - *) - REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) + *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*) + case "$line" in + *'AmqpReplyInbox'*) pending_inbox=$((pending_inbox + 1)) ;; + *'LeadMailbox'*) pending_lead_mailbox=$((pending_lead_mailbox + 1)) ;; + *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; + esac ;; + *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; esac ;; - *'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*) - if [ "$pending_resets" -gt 0 ]; then - pending_resets=$((pending_resets - 1)) + *'AMQP connection recovered; cleared held replies for fresh redelivery'*) + if [ "$pending_inbox" -gt 0 ]; then + pending_inbox=$((pending_inbox - 1)) + REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) + fi + ;; + *'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*) + if [ "$pending_lead_mailbox" -gt 0 ]; then + pending_lead_mailbox=$((pending_lead_mailbox - 1)) REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) fi ;; esac done < "$log_file" - REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending_resets)) + REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending_inbox + pending_lead_mailbox)) } # CB-600: sourceable for testing. When this file is SOURCED (not executed) it stops here — nothing diff --git a/scripts/test-redeploy-fleetd.sh b/scripts/test-redeploy-fleetd.sh index d16b8e1..4ff1d95 100755 --- a/scripts/test-redeploy-fleetd.sh +++ b/scripts/test-redeploy-fleetd.sh @@ -34,20 +34,78 @@ LOG assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "no-errors unexplained" } -test_recovered_connection_error() { - cat > "$TMP/recovered.log" <<'LOG' -2026-09-05 12:00:00 ERROR com.rabbitmq.client.impl.ForgivingExceptionHandler - An unexpected connection driver error occurred (Exception message: Connection reset) -2026-09-05 12:00:01 INFO dev.ltms.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery -LOG - classify_fixture recovered.log - assert_equals 1 "$REDEPLOY_ERROR_COUNT" "recovered total" - assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "recovered AMQP errors" - assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "recovered unexplained" +test_recovery_patterns_match_source() { + grep -F 'AMQP connection recovered; cleared held replies for fresh redelivery' \ + "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/AmqpReplyInbox.java" > /dev/null \ + || fail "reply-inbox recovery pattern no longer matches source" + grep -F 'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery' \ + "$ROOT/fleetd/src/main/java/dev/ltms/fleet/msg/LeadMailbox.java" > /dev/null \ + || fail "lead-mailbox recovery pattern no longer matches source" } -test_unrecovered_connection_error() { +test_attributed_recovered_connection_error() { + cat > "$TMP/attributed-recovered.log" <<'LOG' +2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred +2026-09-05 12:00:01 INFO AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery +LOG + classify_fixture attributed-recovered.log + assert_equals 1 "$REDEPLOY_ERROR_COUNT" "attributed-recovered total" + assert_equals 1 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "attributed-recovered errors" + assert_equals 0 "$REDEPLOY_UNEXPLAINED_ERRORS" "attributed-recovered unexplained" +} + +test_real_error_shape_is_loud_when_unattributable() { + # The real handler ERROR lines name neither inbox nor lead mailbox. Even though both connections + # later recover, this region must stay loud because the recovery cannot be assigned safely. + cat > "$TMP/real-error-shape.log" <<'LOG' +17:37:53.537 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred +18:20:33.027 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! +18:30:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred +19:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! +20:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred +21:00:00.000 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - Caught an exception during connection recovery! +21:41:39.329 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery +21:41:48.693 INFO [AMQP Connection 10.10.20.13:5672] d.l.fleet.msg.AmqpReplyInbox - AMQP connection recovered; cleared held replies for fresh redelivery +LOG + classify_fixture real-error-shape.log + assert_equals 6 "$REDEPLOY_ERROR_COUNT" "real-error-shape total" + assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "real-error-shape recovered" + assert_equals 6 "$REDEPLOY_UNEXPLAINED_ERRORS" "real-error-shape unexplained" +} + +test_cross_connection_unattributable_errors_stay_loud() { + # This is the reported unsafe shape. Both handler lines lack a connection kind, so LeadMailbox + # recoveries must not consume either one. + cat > "$TMP/cross-unattributable.log" <<'LOG' +2026-09-05 12:00:00 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred +2026-09-05 12:00:01 ERROR [AMQP Connection 10.10.20.13:5672] c.r.c.i.ForgivingExceptionHandler - An unexpected connection driver error occurred +2026-09-05 12:00:02 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery +2026-09-05 12:00:03 INFO [AMQP Connection 10.10.20.13:5672] d.ltms.fleet.msg.LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery +LOG + classify_fixture cross-unattributable.log + assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-unattributable total" + assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-unattributable recovered" + assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-unattributable unexplained" +} + +test_attributed_cross_connection_errors_stay_loud() { + # This fixture exercises the per-kind pending state should a future layout include a simple class + # name on the handler line. LeadMailbox recovery cannot heal AmqpReplyInbox errors. + cat > "$TMP/cross-attributed.log" <<'LOG' +2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred +2026-09-05 12:00:01 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred +2026-09-05 12:00:02 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery +2026-09-05 12:00:03 INFO LeadMailbox - AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery +LOG + classify_fixture cross-attributed.log + assert_equals 2 "$REDEPLOY_ERROR_COUNT" "cross-attributed total" + assert_equals 0 "$REDEPLOY_RECOVERED_AMQP_ERRORS" "cross-attributed recovered" + assert_equals 2 "$REDEPLOY_UNEXPLAINED_ERRORS" "cross-attributed unexplained" +} + +test_attributed_unrecovered_connection_error() { cat > "$TMP/unrecovered.log" <<'LOG' -2026-09-05 12:00:00 ERROR com.rabbitmq.client.impl.ForgivingExceptionHandler - An unexpected connection driver error occurred (Exception message: Connection reset) +2026-09-05 12:00:00 ERROR AmqpReplyInbox ForgivingExceptionHandler - An unexpected connection driver error occurred LOG classify_fixture unrecovered.log assert_equals 1 "$REDEPLOY_ERROR_COUNT" "unrecovered total" @@ -65,7 +123,7 @@ LOG assert_equals 1 "$REDEPLOY_UNEXPLAINED_ERRORS" "other-error unexplained" } -test_mutation_is_caught() { +test_recovery_requirement_mutation_is_caught() { classify_amqp_connection_errors() { local log_file="$1" line REDEPLOY_ERROR_COUNT=0 @@ -76,7 +134,7 @@ test_mutation_is_caught() { *' ERROR '*|*' SEVERE '*) REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) case "$line" in - *'com.rabbitmq.client.impl.ForgivingExceptionHandler'*'Connection reset'*) + *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*) REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) ;; *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; @@ -86,17 +144,61 @@ test_mutation_is_caught() { done < "$log_file" } - if test_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then + if test_attributed_unrecovered_connection_error > "$TMP/mutation-output" 2>&1; then fail "mutation accepted an unrecovered connection error" fi grep -F 'FAIL: unrecovered AMQP errors: expected 0, got 1' "$TMP/mutation-output" > /dev/null \ || fail "mutation failed without the expected assertion" - printf 'Mutation check: FAIL: unrecovered AMQP errors: expected 0, got 1\n' + printf 'Recovery mutation: FAIL: unrecovered AMQP errors: expected 0, got 1\n' +} + +test_shared_counter_mutation_is_caught() { + classify_amqp_connection_errors() { + local log_file="$1" line pending=0 + REDEPLOY_ERROR_COUNT=0 + REDEPLOY_RECOVERED_AMQP_ERRORS=0 + REDEPLOY_UNEXPLAINED_ERRORS=0 + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *' ERROR '*|*' SEVERE '*) + REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1)) + case "$line" in + *'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*) + case "$line" in + *'AmqpReplyInbox'*|*'LeadMailbox'*) pending=$((pending + 1)) ;; + *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; + esac + ;; + *) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;; + esac + ;; + *'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*) + if [ "$pending" -gt 0 ]; then + pending=$((pending - 1)) + REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1)) + fi + ;; + esac + done < "$log_file" + REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending)) + } + + if test_attributed_cross_connection_errors_stay_loud > "$TMP/shared-mutation-output" 2>&1; then + fail "shared counter mutation accepted cross-connection recovery" + fi + grep -F 'FAIL: cross-attributed recovered: expected 0, got 2' "$TMP/shared-mutation-output" > /dev/null \ + || fail "shared counter mutation failed without the expected assertion" + printf 'Shared-counter mutation: FAIL: cross-attributed recovered: expected 0, got 2\n' } test_no_errors -test_recovered_connection_error -test_unrecovered_connection_error +test_recovery_patterns_match_source +test_attributed_recovered_connection_error +test_real_error_shape_is_loud_when_unattributable +test_cross_connection_unattributable_errors_stay_loud +test_attributed_cross_connection_errors_stay_loud +test_attributed_unrecovered_connection_error test_other_error_is_unexplained -test_mutation_is_caught +test_recovery_requirement_mutation_is_caught +test_shared_counter_mutation_is_caught printf 'PASS: redeploy log classifier\n'