From 4a5030a5c63cbdcdaf0f6d26116ba7597d870793 Mon Sep 17 00:00:00 2001 From: Dai Ha Date: Fri, 4 Sep 2026 16:57:36 +0700 Subject: [PATCH] #348: drop a chrome skip that cannot fire, and pin the live pattern shape --- .../ltms/fleet/inject/CompletionResolver.java | 55 +++++++++++++------ .../fleet/inject/CompletionResolverTest.java | 28 ++++++++++ 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java index 0837ebd..50755fc 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java +++ b/fleetd/src/main/java/dev/ltms/fleet/inject/CompletionResolver.java @@ -618,30 +618,38 @@ public final class CompletionResolver implements TurnListener { } /** - * True when the pattern appears before the first sentence ending in the matched pane line, rather - * than after a member has started prose about it. + * True when nothing before the match on this pane line ends a sentence — that is, the match is + * still inside the line's first sentence rather than inside prose a member wrote about it. + * Used to decide whether an exhaustion match may quarantine a credential (fleetd #348). * - *

Leading terminal chrome is skipped first — box-drawing characters, bullets, gutter bars and - * spaces. Exhaustion patterns often name only the decisive words in a provider message, such as - * {@code "usage limit has been reached"}; they do not include its leading {@code "The"}. A bare - * {@code lookingAt} would therefore reject that genuine refusal, including one behind terminal - * chrome. + *

Why this is looser than {@link #startsWithBackendError}. An + * {@code exhaustedPattern} is written per profile and may name only the decisive words of a + * provider message — {@code "usage limit has been reached"} without its leading {@code "The"}. + * A start-of-line check would then reject the genuine refusal. That is the false negative + * fleetd #348's invariant 1 calls the worse direction: an unrecorded exhaustion leaves the + * fleet spawning into a credential with no capacity, and a quarantine runs 1800s against the + * backend-error cooldown's fixed 60s. * - *

A member's prose about a refusal normally follows a completed sentence. That sentence ending - * is enough to keep it from reaching a cooldown sink while the send still fails with the full pane - * tail. This is deliberately less strict than the backend-error check because exhausted patterns - * may start after a provider's leading words. + *

This rule accepts a superset of what a start-of-line check accepts: if the match begins + * right after the chrome, there is nothing in front of it, so there is no sentence ending + * either. So moving to it cannot add a false negative. + * + *

No chrome skipping here, deliberately. The first version of this method + * copied {@code startsWithBackendError}'s leading-chrome loop. Measured on merge: deleting that + * loop left all 1369 tests green, and it must — the scan only looks for {@code . ! ?}, and no + * terminal chrome character is one of those. A step that cannot change the result is worse than + * no step, because the next reader takes it as evidence that chrome was handled. + * + *

It stays a heuristic. Prose whose first sentence carries the pattern still + * notifies the sink, and a genuine refusal behind an earlier full stop (a hostname, a version + * number) still does not. Both are known and neither is fixed here. */ private static boolean startsWithExhaustion(String line, Pattern pattern) { - int i = 0; - while (i < line.length() && !Character.isLetterOrDigit(line.charAt(i))) { - i++; - } var matcher = pattern.matcher(line); if (!matcher.find()) { return false; } - for (int prefix = i; prefix < matcher.start(); prefix++) { + for (int prefix = 0; prefix < matcher.start(); prefix++) { if (".!?".indexOf(line.charAt(prefix)) >= 0) { return false; } @@ -649,7 +657,20 @@ public final class CompletionResolver implements TurnListener { return true; } - /** True when an error pattern begins the matched pane line after optional terminal chrome. */ + /** + * True when the error pattern begins the matched pane line, rather than appearing in prose. + * + *

Leading terminal chrome is skipped first — box-drawing characters, bullets, gutter bars and + * spaces. #339 introduced this check with a bare {@code lookingAt}, and that rejected a genuine + * error line rendered as {@code "| 503 Service Unavailable: ..."}: the send still failed, but the + * credential outage was never recorded. That is the false negative #339's own invariant 3 called + * worse than the false positive it set out to fix — measured with a throwaway probe on the + * raw-scrape path, which is exactly the path whose comment says to expect leading chrome. + * + *

Skipping only a leading run of non-letter, non-digit characters keeps the fix's intent. A + * member's prose ({@code "I checked the retry path. An API Error: makes it back off."}) still + * does not match, because there the pattern sits after words, not after chrome. + */ private static boolean startsWithBackendError(String line, Pattern pattern) { int i = 0; while (i < line.length() && !Character.isLetterOrDigit(line.charAt(i))) { diff --git a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java index 8e73b5d..e1aaab2 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/inject/CompletionResolverTest.java @@ -574,6 +574,34 @@ class CompletionResolverTest { "a real exhaustion behind terminal chrome must reach the sink"); } + /** + * The live fleet configures {@code exhaustedPattern: "The usage limit has been reached"} — with + * the leading {@code "The"}. Every other test here uses a pattern without it, which is the shape + * that made fleetd #348 need a looser rule than a start-of-line check. This pins the deployed + * shape as well, so a later tightening of {@link CompletionResolver} cannot silently stop + * recording the exhaustion this fleet actually reports. + * + *

What it does not prove: that this is the only pattern shape an operator will write. + */ + @Test + void anExhaustionPatternCarryingItsLeadingWordsStillNotifiesTheSink() { + String block = "⏺ │ The usage limit has been reached. Try again later.\n❯ "; + FakeHerdr herdr = new FakeHerdr().readText(block); + Rendezvous rendezvous = new Rendezvous(); + ExhaustedPatternLookup patterns = target -> Pattern.compile("The usage limit has been reached"); + java.util.List notified = new java.util.ArrayList<>(); + ExhaustionSink sink = (target, reason, profile) -> notified.add(target + ": " + reason); + CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous, patterns, sink); + + var waiter = rendezvous.open("term_a"); + resolver.resolve("term_a", new CompletionResolver.InFlight(waiter, null)); + + assertEquals(Rendezvous.Kind.BACKEND_EXHAUSTED, waiter.getNow(null).kind(), + "the live pattern shape must still fail the send as exhausted"); + assertEquals(1, notified.size(), + "the live pattern shape must still reach the sink"); + } + @Test void aLosingBackendExhaustedClassificationNeverNotifiesTheExhaustionSink() { // The waiter was already resolved (e.g. by the worker's own reply) before this scrape landed —