#164: classify a backend-error scrape as WORKER_FAILED, carrying the whole pane
CI / contract (push) Successful in 48s
CI / build (push) Successful in 1m22s

main already shipped the core of #164 in 3bfa828: the MIN_TURN_NANOS floor and
the hard fail on an empty or unreadable scrape. This adds the one case that was
still resolving as a success -- a scrape that reads cleanly but whose content is
the backend's own rejection (e.g. "API Error: 400 invalid request body").

The BACKEND_ERROR pattern is deliberately narrow. A growing list of ad-hoc error
strings rots as backends change their wording, and broader backend-error
surfacing is #164 point 3.

Because the pattern is a heuristic, it also matches a member that forgot
fleet_reply while reporting *about* a backend error. So the failure reason
carries the whole pane tail, not just the matched line: a genuine backend error
reads as before, and a false positive keeps its report instead of losing it.

Checked before merging: 3bfa828 is an ancestor of main; the branch was current
with main; mvn clean install green unpiped (1037 tests, 0 [ERROR] lines); and
each of the 5 new tests fails with the fix commented out.

Co-authored-by: fleetd worker <worker@ltms.dev>
This commit is contained in:
Dai Ha
2026-08-31 10:56:40 +07:00
parent a814d1ef00
commit 4ac688b6d9
3 changed files with 111 additions and 0 deletions
@@ -74,6 +74,14 @@ public final class CompletionResolver implements TurnListener {
*/
public static final long MIN_TURN_NANOS = Duration.ofSeconds(2).toNanos();
/**
* fleetd#164 (part 2): one stable, explicit backend-failure marker seen on a Claude Code pane
* when the backend itself rejected the turn (e.g. {@code "API Error: 400 invalid request body"}).
* Kept deliberately narrow — a growing list of ad-hoc error strings rots as backends change their
* wording; broader backend-error surfacing is out of scope here (fleetd#164 point 3).
*/
private static final Pattern BACKEND_ERROR = Pattern.compile("(?i)\\bAPI Error\\s*:");
private static final String CLIPPED_PANE_TAIL_MARKER =
"[Pane tail clipped: member did not call fleet_reply.]";
@@ -278,6 +286,20 @@ public final class CompletionResolver implements TurnListener {
}
return;
}
// fleetd#164 (part 2): a scrape that read cleanly and produced content still isn't a real
// reply when that content is the backend's own rejection (e.g. an HTTP 400 before the worker
// did any work). Classify it as a failure naming the member, rather than handing the caller a
// scrape that reads like a completed answer.
String backendError = firstMatchingLine(assistantBlock, BACKEND_ERROR);
if (backendError != null) {
// Carry the whole scrape, not just the matched line. The pattern is a heuristic: a member
// that forgot fleet_reply while reporting *about* a backend error matches it too. Failing
// is still right — the caller must not read a scrape as an answer — but dropping the rest
// of the pane would destroy the report, which is the same defect fleetd#164 is about.
fail(target, turn, "member " + target + " ended on a backend error: " + backendError
+ "\n--- pane tail ---\n" + tail);
return;
}
String completion = clipped ? tail + "\n" + CLIPPED_PANE_TAIL_MARKER : tail;
if (rendezvous.resolveCompletion(waiter, completion)) {
inFlight.remove(target, turn);
@@ -566,6 +566,73 @@ class CompletionResolverTest {
assertEquals("The usage limit has been reached.", waiter.getNow(null).text());
}
// --- fleetd#164 (part 2 addendum): narrow BACKEND_ERROR pattern classification ---------
@Test
void classifiesABackendErrorLineAsAFailureInsteadOfACompletedReply() {
String block = "⏺ API Error: 400 invalid request body\n❯ ";
FakeHerdr herdr = new FakeHerdr().readText(block);
Rendezvous rendezvous = new Rendezvous();
CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none());
var waiter = rendezvous.open("term_a");
resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null));
assertTrue(waiter.isDone(), "a backend-error scrape still resolves the blocked send");
assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind(),
"a backend rejection is a failure, not a completed reply");
}
@Test
void theBackendErrorReasonNamesTheMemberAndCarriesTheMatchedLine() {
String block = "⏺ API Error: 400 invalid request body\n❯ ";
FakeHerdr herdr = new FakeHerdr().readText(block);
Rendezvous rendezvous = new Rendezvous();
CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none());
var waiter = rendezvous.open("term_a");
resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null));
String reason = waiter.getNow(null).text();
assertTrue(reason.contains("term_a"), "the failure names the member: " + reason);
assertTrue(reason.contains("API Error: 400 invalid request body"),
"the failure carries the matched backend-error line: " + reason);
}
@Test
void aCaseInsensitiveApiErrorLineIsStillClassifiedAsABackendError() {
String block = "⏺ api error: rate limited\n❯ ";
FakeHerdr herdr = new FakeHerdr().readText(block);
Rendezvous rendezvous = new Rendezvous();
CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none());
var waiter = rendezvous.open("term_a");
resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null));
assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind(), "the pattern is case-insensitive");
}
@Test
void aBackendErrorFailureStillCarriesTheRestOfTheScrape() {
// The pattern is a heuristic: a member that forgot fleet_reply while *reporting on* a backend
// error matches it too. Failing is still correct, but the report itself must survive — losing
// it would be the same information-destroying defect fleetd#164 exists to fix.
String block = "\u23fa I looked into the gateway problem.\n"
+ "The log line was: API Error: 400 invalid request body\n"
+ "The cause is a missing content-type header.\n\u276f ";
FakeHerdr herdr = new FakeHerdr().readText(block);
Rendezvous rendezvous = new Rendezvous();
CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, ExhaustedPatternLookup.none(), ExhaustionSink.none());
var waiter = rendezvous.open("term_a");
resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null));
String reason = waiter.getNow(null).text();
assertEquals(Rendezvous.Kind.FAILED, waiter.getNow(null).kind());
assertTrue(reason.contains("The cause is a missing content-type header."),
"the failure carries the rest of the pane, not only the matched line: " + reason);
}
@Test
void coverageIsOffWhenNoProfileHasAPatternConfigured() {
assertEquals("off (no profile has an exhaustedPattern configured; profiles: [terra])",
@@ -86,6 +86,28 @@ class MessageServiceTest {
assertTrue(reply.completed(), "a scraped completion still counts as completed");
}
@Test
void backendErrorScrapeThroughMessageServiceFailsInsteadOfBecomingReplyText() throws Exception {
// fleetd#164 (part 2 addendum): a scrape that reads cleanly but is only the backend's own
// rejection (e.g. an HTTP 400) must reach the caller as WORKER_FAILED, not as a completed
// reply whose text happens to be the error line.
CompletableFuture<MessageService.Reply> send = sendAsync();
awaitWaiting();
herdr.readText("$ prompt");
injector.onStatus(T, AgentStatus.IDLE);
injector.onStatus(T, AgentStatus.WORKING);
herdr.readText("⏺ API Error: 400 invalid request body");
injector.onStatus(T, AgentStatus.IDLE);
MessageService.Reply reply = send.get(5, TimeUnit.SECONDS);
assertEquals(MessageService.Outcome.WORKER_FAILED, reply.outcome(),
"a backend rejection must use the caller's failure outcome, not a completed reply");
assertFalse(reply.completed(), "plain backend errors are never fallback reply content");
assertTrue(reply.text().contains("API Error: 400 invalid request body"),
"the visible backend error is carried as the failure reason: " + reply.text());
}
@Test
void explicitFleetReplyResolvesAsReplied() throws Exception {
CompletableFuture<MessageService.Reply> send = sendAsync();