fleet/fleetd
3
0
Fork 0
Code Issues 78 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Keep unattributed AMQP errors loud
CI / contract (pull_request) Successful in 52s
Details
CI / build (pull_request) Successful in 1m30s
Details

Browse Source
This commit is contained in:
Dai Ha
2026-09-05 05:36:53 +07:00
parent e4973eb8a4
commit 0241e0d3a8
2 changed files with 143 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/redeploy-fleetd.sh
+23 -14
View File
@@ -116,12 +116,13 @@ check_log_path_matches_plist() {
ok "log path check: script and plist agree ($resolved_out)"
}
# Classify ERROR lines in one fresh log region. RabbitMQ's client reports a lost TCP connection
# through ForgivingExceptionHandler. We only quiet its exact "Connection reset" error after one of
# fleetd's own recovery listeners later reports a completed AMQP recovery. Every other ERROR, and
# every unmatched reset, stays unexplained so a failed broker link remains loud.
# Classify ERROR lines in one fresh log region. The logging layout abbreviates logger packages, so
# match simple class names and the handler's actual ERROR messages. An error can be quiet only when
# its own line names AmqpReplyInbox or LeadMailbox; the handler lines now seen in fleetd.out name
# neither connection, so they deliberately stay unexplained. This avoids letting one connection's
# recovery hide a failure in the other connection.
classify_amqp_connection_errors() {
local log_file="$1" line pending_resets=0
local log_file="$1" line pending_inbox=0 pending_lead_mailbox=0
REDEPLOY_ERROR_COUNT=0
REDEPLOY_RECOVERED_AMQP_ERRORS=0
REDEPLOY_UNEXPLAINED_ERRORS=0
@@ -131,24 +132,32 @@ classify_amqp_connection_errors() {
*' ERROR '*|*' SEVERE '*)
REDEPLOY_ERROR_COUNT=$((REDEPLOY_ERROR_COUNT + 1))
case "$line" in
*'com.rabbitmq.client.impl.ForgivingExceptionHandler'*'Connection reset'*)
pending_resets=$((pending_resets + 1))
;;
*)
REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1))
*'ForgivingExceptionHandler'*'An unexpected connection driver error occurred'*|*'ForgivingExceptionHandler'*'Caught an exception during connection recovery!'*)
case "$line" in
*'AmqpReplyInbox'*) pending_inbox=$((pending_inbox + 1)) ;;
*'LeadMailbox'*) pending_lead_mailbox=$((pending_lead_mailbox + 1)) ;;
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
esac
;;
*) REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + 1)) ;;
esac
;;
*'AMQP connection recovered; cleared held replies for fresh redelivery'*|*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*)
if [ "$pending_resets" -gt 0 ]; then
pending_resets=$((pending_resets - 1))
*'AMQP connection recovered; cleared held replies for fresh redelivery'*)
if [ "$pending_inbox" -gt 0 ]; then
pending_inbox=$((pending_inbox - 1))
REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
fi
;;
*'AMQP lead mailbox connection recovered; cleared held messages for fresh redelivery'*)
if [ "$pending_lead_mailbox" -gt 0 ]; then
pending_lead_mailbox=$((pending_lead_mailbox - 1))
REDEPLOY_RECOVERED_AMQP_ERRORS=$((REDEPLOY_RECOVERED_AMQP_ERRORS + 1))
fi
;;
esac
done < "$log_file"
REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending_resets))
REDEPLOY_UNEXPLAINED_ERRORS=$((REDEPLOY_UNEXPLAINED_ERRORS + pending_inbox + pending_lead_mailbox))
}
# CB-600: sourceable for testing. When this file is SOURCED (not executed) it stops here — nothing