diff --git a/fleetd/src/main/java/dev/ltms/fleet/inject/BackendErrorSink.java b/fleetd/src/main/java/dev/ltms/fleet/inject/BackendErrorSink.java index 3370afd..df6bd17 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/inject/BackendErrorSink.java +++ b/fleetd/src/main/java/dev/ltms/fleet/inject/BackendErrorSink.java @@ -1,10 +1,12 @@ package dev.ltms.fleet.inject; /** - * Notified when {@link CompletionResolver} actually delivers a typed backend-error classification - * to a waiting send (fleetd#201 / #227) — never on a race that lost. {@link CompletionResolver} - * calls this only after {@code Rendezvous.resolveFailure} returns {@code true} for that exact - * waiter, mirroring the win-only race rule {@link ExhaustionSink} already uses. + * Notified when {@link CompletionResolver} has a backend-error match at the start of a pane line, + * or both a match and its too-fast crash signature, for a waiting send (fleetd#201 / #227). A text + * match inside ordinary pane prose can be a member's report about an error, so it fails the send + * without notifying this sink. + * {@link CompletionResolver} calls this only after {@code Rendezvous.resolveFailure} returns + * {@code true} for that exact waiter, mirroring the win-only race rule {@link ExhaustionSink} uses. * *

The public send result is unchanged by this classification — it is still a failed send * ({@code Rendezvous.Kind#FAILED}); this sink is the internal seam a later stage (fleetd#201 Unit diff --git a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java index c806fd8..a1787e6 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java +++ b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java @@ -387,9 +387,10 @@ public final class CompletionResolver implements TurnListener { // fleetd#164 (part 2) / fleetd#201: a scrape that read cleanly and produced content still // isn't a real reply when that content is the backend's own rejection (e.g. an HTTP 400 // before the worker did any work). Classify it as a failure naming the member, rather than - // handing the caller a scrape that reads like a completed answer, and — only on the - // resolution that actually wins the race, mirroring the exhaustion sink above — notify the - // typed backend-error sink so a later stage can act on repeated failures. + // handing the caller a scrape that reads like a completed answer. A text match alone is not + // enough to notify the typed backend-error sink: this assistant block can be a member's + // normal prose about an error. A line that starts with the error match is stronger evidence; + // the too-fast path below also has its crash signature before it records a credential failure. String backendError = firstMatchingLine(assistantBlock, backendErrorPatternOrFallback(target)); if (backendError != null) { // Carry the whole scrape, not just the matched line. The pattern is a heuristic: a member @@ -401,9 +402,9 @@ public final class CompletionResolver implements TurnListener { if (rendezvous.resolveFailure(waiter, reason)) { inFlight.remove(target, turn); log.warn("failing send to {} via turn-stall fallback: {}", target, reason); - // fleetd#201 Unit 1: only on the resolution that actually won the race — a late - // duplicate must never double-count one backend failure. - backendErrorSink.onBackendError(target, backendError, reason); + if (startsWithBackendError(backendError, backendErrorPatternOrFallback(target))) { + backendErrorSink.onBackendError(target, backendError, reason); + } } return; } @@ -492,8 +493,9 @@ public final class CompletionResolver implements TurnListener { if (rendezvous.resolveFailure(waiter, reason)) { inFlight.remove(target, turn); log.warn("failing send to {} via turn-stall fallback from the raw scrape: {}", target, reason); - // fleetd#201 Unit 1: only on the resolution that actually won the race. - backendErrorSink.onBackendError(target, backendError, reason); + if (startsWithBackendError(backendError, backendErrorPatternOrFallback(target))) { + backendErrorSink.onBackendError(target, backendError, reason); + } } return true; } @@ -540,10 +542,10 @@ public final class CompletionResolver implements TurnListener { * {@link #MIN_TURN_NANOS} — a crash signature (e.g. a backend HTTP 400 before the worker did * anything) that a bare {@code BUSY -> DONE} transition cannot be told apart from a genuinely * fast completion. Runs the same backend-error classification the normal and raw-scrape paths - * apply, against whatever is on screen right now: a match is a typed failure that notifies - * {@link #backendErrorSink} (only on the resolution that wins the race); a non-match stays the - * original generic too-fast failure, naming the member and both timings, with whatever the pane - * shows appended so the caller sees the cause, not just "it failed". + * apply, against whatever is on screen right now: a match together with the too-fast crash + * signature notifies {@link #backendErrorSink} (only on the resolution that wins the race). A + * non-match stays the original generic too-fast failure, naming the member and both timings, + * with whatever the pane shows appended so the caller sees the cause, not just "it failed". */ private void failTooFast(String target, InFlight turn, CompletableFuture waiter, long elapsedNanos) { @@ -611,6 +613,11 @@ public final class CompletionResolver implements TurnListener { return null; } + /** True when the error pattern begins the matched pane line, rather than appearing in prose. */ + private static boolean startsWithBackendError(String line, Pattern pattern) { + return pattern.matcher(line).lookingAt(); + } + /** * Coverage summary for the CB-578 stage A exhausted-pattern classification, logged at startup * the way {@link dev.ltms.fleet.health.FleetHealthMonitor#coverage} is — so an operator can diff --git a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java index e056b6b..45e7efd 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java @@ -804,7 +804,28 @@ class CompletionResolverTest { // --- fleetd#201 Unit 1: target-keyed backend-error pattern + typed sink ---------------------- @Test - void aConfiguredBackendErrorPatternClassifiesAMatchAsAFailureAndNotifiesTheSinkOnce() { + void aNormalMemberReportMentioningTheFallbackErrorPatternFailsButDoesNotNotifyTheSink() { + String block = "⏺ I checked the retry path. An API Error: makes it back off.\n❯ "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + java.util.List notified = new java.util.ArrayList<>(); + BackendErrorSink sink = (target, matchedLine, reason) -> notified.add(target + ": " + matchedLine); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, + ExhaustedPatternLookup.none(), ExhaustionSink.none(), BackendErrorPatternLookup.legacy(), sink); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind(), + "a scrape mentioning the pattern must still fail the send"); + assertTrue(waiter.getNow(null).text().contains("I checked the retry path. An API Error: makes it back off."), + "the failure must keep the whole pane tail"); + assertTrue(notified.isEmpty(), + "a normal report mentioning the fallback pattern must not record a credential failure"); + } + + @Test + void aConfiguredBackendErrorPatternAtTheStartOfALineClassifiesAMatchAndNotifiesTheSinkOnce() { String block = "⏺ 503 Service Unavailable: upstream credential rejected\n❯ "; FakeHerdr herdr = new FakeHerdr().readText(block); Rendezvous rendezvous = new Rendezvous(); @@ -924,7 +945,7 @@ class CompletionResolverTest { } @Test - void aConfiguredPatternAlsoClassifiesTheRawScrapeFallbackAndNotifiesTheSink() { + void aConfiguredPatternAtTheStartOfALineAlsoClassifiesTheRawScrapeFallbackAndNotifiesTheSink() { // No ⏺ marker and leading TUI chrome ⇒ lastAssistantBlock() yields "", so classification must // fall back to the raw scrape (fleetd#211) — and it must use the configured pattern too. String block = """