Classify named AMQP recovery errors
CI / contract (pull_request) Successful in 51s
CI / build (pull_request) Successful in 2m16s

This commit is contained in:
Dai Ha
2026-09-05 06:05:52 +07:00
parent 29cd1194c2
commit 09159f2857
2 changed files with 76 additions and 42 deletions
+11 -11
View File
@@ -116,11 +116,9 @@ check_log_path_matches_plist() {
ok "log path check: script and plist agree ($resolved_out)"
}
# Classify ERROR lines in one fresh log region. The logging layout abbreviates logger packages, so
# match simple class names and the handler's actual ERROR messages. An error can be quiet only when
# its own line names AmqpReplyInbox or LeadMailbox; the handler lines now seen in fleetd.out name
# neither connection, so they deliberately stay unexplained. This avoids letting one connection's
# recovery hide a failure in the other connection.
# Classify ERROR lines in one fresh log region. AMQP failure messages now include the connection
# name, so a recovery can clear only errors for its own connection. A candidate with neither name
# remains unexplained: it must never be quieted by a recovery on the other connection.
classify_amqp_connection_errors() {
local log_file="$1" line pending_inbox=0 pending_lead_mailbox=0
REDEPLOY_ERROR_COUNT=0
@@ -132,12 +130,14 @@ classify_amqp_connection_errors() {
*' ERROR '*|*' SEVERE '*)
REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
case "$line" in
*'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*)
case "$line" in
*'AmqpReplyInbox'*) pending_inbox=$((pending_inbox + 1)) ;;
*'LeadMailbox'*) pending_lead_mailbox=$((pending_lead_mailbox + 1)) ;;
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
esac
*'AMQP connection fleetd-reply-inbox: An unexpected connection driver error occurred'*|*'AMQP connection fleetd-reply-inbox: Caught an exception during connection recovery!'*)
pending_inbox=$((pending_inbox + 1))
;;
*'AMQP connection fleetd-lead-mailbox: An unexpected connection driver error occurred'*|*'AMQP connection fleetd-lead-mailbox: Caught an exception during connection recovery!'*)
pending_lead_mailbox=$((pending_lead_mailbox + 1))
;;
*'AMQP connection'*'An unexpected connection driver error occurred'*|*'AMQP connection'*'Caught an exception during connection recovery!'*)
REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1))
;;
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
esac