From 4ac688b6d9eb4d8bc0b9377131bdf939b0eaea06 Mon Sep 17 00:00:00 2001 From: Dai Ha Date: Mon, 31 Aug 2026 10:56:40 +0700 Subject: [PATCH] #164: classify a backend-error scrape as WORKER_FAILED, carrying the whole pane main already shipped the core of #164 in 3bfa828: the MIN_TURN_NANOS floor and the hard fail on an empty or unreadable scrape. This adds the one case that was still resolving as a success -- a scrape that reads cleanly but whose content is the backend's own rejection (e.g. "API Error: 400 invalid request body"). The BACKEND_ERROR pattern is deliberately narrow. A growing list of ad-hoc error strings rots as backends change their wording, and broader backend-error surfacing is #164 point 3. Because the pattern is a heuristic, it also matches a member that forgot fleet_reply while reporting *about* a backend error. So the failure reason carries the whole pane tail, not just the matched line: a genuine backend error reads as before, and a false positive keeps its report instead of losing it. Checked before merging: 3bfa828 is an ancestor of main; the branch was current with main; mvn clean install green unpiped (1037 tests, 0 [ERROR] lines); and each of the 5 new tests fails with the fix commented out. Co-authored-by: fleetd worker --- .../ltms/fleet/inject/CompletionResolver.java | 22 ++++++ .../fleet/inject/CompletionResolverTest.java | 67 +++++++++++++++++++ .../ltms/fleet/msg/MessageServiceTest.java | 22 ++++++ 3 files changed, 111 insertions(+) diff --git a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java index c1dd7af..94adcc9 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java +++ b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java @@ -74,6 +74,14 @@ public final class CompletionResolver implements TurnListener { */ public static final long MIN_TURN_NANOS = Duration.ofSeconds(2).toNanos(); + /** + * fleetd#164 (part 2): one stable, explicit backend-failure marker seen on a Claude Code pane + * when the backend itself rejected the turn (e.g. {@code "API Error: 400 invalid request body"}). + * Kept deliberately narrow — a growing list of ad-hoc error strings rots as backends change their + * wording; broader backend-error surfacing is out of scope here (fleetd#164 point 3). + */ + private static final Pattern BACKEND_ERROR = Pattern.compile("(?i)\\bAPI Error\\s*:"); + private static final String CLIPPED_PANE_TAIL_MARKER = "[Pane tail clipped: member did not call fleet_reply.]"; @@ -278,6 +286,20 @@ public final class CompletionResolver implements TurnListener { } return; } + // fleetd#164 (part 2): a scrape that read cleanly and produced content still isn't a real + // reply when that content is the backend's own rejection (e.g. an HTTP 400 before the worker + // did any work). Classify it as a failure naming the member, rather than handing the caller a + // scrape that reads like a completed answer. + String backendError = firstMatchingLine(assistantBlock, BACKEND_ERROR); + if (backendError != null) { + // Carry the whole scrape, not just the matched line. The pattern is a heuristic: a member + // that forgot fleet_reply while reporting *about* a backend error matches it too. Failing + // is still right — the caller must not read a scrape as an answer — but dropping the rest + // of the pane would destroy the report, which is the same defect fleetd#164 is about. + fail(target, turn, "member " + target + " ended on a backend error: " + backendError + + "\n--- pane tail ---\n" + tail); + return; + } String completion = clipped ? tail + "\n" + CLIPPED_PANE_TAIL_MARKER : tail; if (rendezvous.resolveCompletion(waiter, completion)) { inFlight.remove(target, turn); diff --git a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java index 407e163..3f7e3c0 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java @@ -566,6 +566,73 @@ class CompletionResolverTest { assertEquals("The usage limit has been reached.", waiter.getNow(null).text()); } + // --- fleetd#164 (part 2 addendum): narrow BACKEND_ERROR pattern classification --------- + + @Test + void classifiesABackendErrorLineAsAFailureInsteadOfACompletedReply() { + String block = "⏺ API Error: 400 invalid request body\n❯ "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none()); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + assertTrue(waiter.isDone(), "a backend-error scrape still resolves the blocked send"); + assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind(), + "a backend rejection is a failure, not a completed reply"); + } + + @Test + void theBackendErrorReasonNamesTheMemberAndCarriesTheMatchedLine() { + String block = "⏺ API Error: 400 invalid request body\n❯ "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none()); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + String reason = waiter.getNow(null).text(); + assertTrue(reason.contains("term_a"), "the failure names the member: " + reason); + assertTrue(reason.contains("API Error: 400 invalid request body"), + "the failure carries the matched backend-error line: " + reason); + } + + @Test + void aCaseInsensitiveApiErrorLineIsStillClassifiedAsABackendError() { + String block = "⏺ api error: rate limited\n❯ "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none()); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind(), "the pattern is case-insensitive"); + } + + @Test + void aBackendErrorFailureStillCarriesTheRestOfTheScrape() { + // The pattern is a heuristic: a member that forgot fleet_reply while *reporting on* a backend + // error matches it too. Failing is still correct, but the report itself must survive — losing + // it would be the same information-destroying defect fleetd#164 exists to fix. + String block = "\u23fa I looked into the gateway problem.\n" + + "The log line was: API Error: 400 invalid request body\n" + + "The cause is a missing content-type header.\n\u276f "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none()); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + String reason = waiter.getNow(null).text(); + assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind()); + assertTrue(reason.contains("The cause is a missing content-type header."), + "the failure carries the rest of the pane, not only the matched line: " + reason); + } + @Test void coverageIsOffWhenNoProfileHasAPatternConfigured() { assertEquals("off (no profile has an exhaustedPattern configured; profiles: [terra])", diff --git a/fleetd/src/test/java/dev/ltms/fleet/msg/MessageServiceTest.java b/fleetd/src/test/java/dev/ltms/fleet/msg/MessageServiceTest.java index dde5d87..eca9721 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/msg/MessageServiceTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/msg/MessageServiceTest.java @@ -86,6 +86,28 @@ class MessageServiceTest { assertTrue(reply.completed(), "a scraped completion still counts as completed"); } + @Test + void backendErrorScrapeThroughMessageServiceFailsInsteadOfBecomingReplyText() throws Exception { + // fleetd#164 (part 2 addendum): a scrape that reads cleanly but is only the backend's own + // rejection (e.g. an HTTP 400) must reach the caller as WORKER_FAILED, not as a completed + // reply whose text happens to be the error line. + CompletableFuture send = sendAsync(); + awaitWaiting(); + + herdr.readText("$ prompt"); + injector.onStatus(T, AgentStatus.IDLE); + injector.onStatus(T, AgentStatus.WORKING); + herdr.readText("⏺ API Error: 400 invalid request body"); + injector.onStatus(T, AgentStatus.IDLE); + + MessageService.Reply reply = send.get(5, TimeUnit.SECONDS); + assertEquals(MessageService.Outcome.WORKER_FAILED, reply.outcome(), + "a backend rejection must use the caller's failure outcome, not a completed reply"); + assertFalse(reply.completed(), "plain backend errors are never fallback reply content"); + assertTrue(reply.text().contains("API Error: 400 invalid request body"), + "the visible backend error is carried as the failure reason: " + reply.text()); + } + @Test void explicitFleetReplyResolvesAsReplied() throws Exception { CompletableFuture send = sendAsync();