CB-109: fail a delegation whose worker wedges in an unknown state
Dogfood found the gap: a turn that dies into an error screen herdr reports as 'unknown' (e.g. the worker hitting API ENOTFOUND) never produces a working->idle boundary, so CB-106 never fires and the async send rides its full 30-min timeout. The injector now counts consecutive 'unknown' samples while a delegation is outstanding; any working/idle sample resets the streak, so only a genuine wedge (~30s continuous unknown) trips it. It then fires TurnListener.onTurnFailed; CompletionResolver scrapes the error screen and resolves the send via Rendezvous.resolveFailure (Kind.FAILED -> Outcome.WORKER_FAILED), surfaced as async phase=failed / REST status=failed / a [worker failed] MCP note, with the error context as the reason. This also frees a delivery that wedged before pickup, which the injectable-only pickup grace could never release.
This commit is contained in:
@@ -21,4 +21,16 @@ class CompletionResolverTest {
|
||||
assertFalse(herdr.called("agent.read"),
|
||||
"a turn nobody is blocked on must not cost a transcript scrape");
|
||||
}
|
||||
|
||||
@Test
|
||||
void failSkipsTheScrapeWhenNoSendIsWaiting() {
|
||||
FakeHerdr herdr = new FakeHerdr();
|
||||
Rendezvous rendezvous = new Rendezvous(); // no waiter opened
|
||||
CompletionResolver resolver = new CompletionResolver(new AgentControl(herdr), rendezvous);
|
||||
|
||||
resolver.fail("term_a");
|
||||
|
||||
assertFalse(herdr.called("agent.read"),
|
||||
"a wedge nobody is blocked on must not cost a transcript scrape");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,6 +172,55 @@ class InjectorTest {
|
||||
assertEquals(List.of(), completed, "no completion is synthesized from an unconfirmed turn");
|
||||
}
|
||||
|
||||
/** Captures both turn-lifecycle callbacks so the CB-109 stall path can be asserted. */
|
||||
private static final class Captor implements TurnListener {
|
||||
final List<String> completed = new ArrayList<>();
|
||||
final List<String> failed = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void onTurnComplete(String target) {
|
||||
completed.add(target);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onTurnFailed(String target) {
|
||||
failed.add(target);
|
||||
}
|
||||
}
|
||||
|
||||
// ~30s of unknown at the 250ms prod poll interval; enough onStatus samples to trip the stall.
|
||||
private static final int STALL_SAMPLES = 130;
|
||||
|
||||
@Test
|
||||
void failsAnOutstandingDelegationWhoseWorkerWedgesInUnknown() {
|
||||
Captor cap = new Captor();
|
||||
Injector inj = new Injector(new AgentControl(herdr), cap);
|
||||
inj.enqueue(T, "task");
|
||||
|
||||
inj.onStatus(T, AgentStatus.IDLE); // deliver
|
||||
inj.onStatus(T, AgentStatus.WORKING); // worker starts the turn
|
||||
for (int i = 0; i < STALL_SAMPLES; i++) inj.onStatus(T, AgentStatus.UNKNOWN); // then wedges
|
||||
|
||||
assertEquals(List.of(T), cap.failed, "a sustained unknown streak fails the outstanding send");
|
||||
assertEquals(List.of(), cap.completed, "a wedge is a failure, not a completion");
|
||||
assertTrue(inj.activeTargets().isEmpty(), "the wedged target is reclaimed, not polled forever");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aTransientUnknownGlitchNeitherFailsNorBlocksCompletion() {
|
||||
Captor cap = new Captor();
|
||||
Injector inj = new Injector(new AgentControl(herdr), cap);
|
||||
inj.enqueue(T, "task");
|
||||
|
||||
inj.onStatus(T, AgentStatus.IDLE); // deliver
|
||||
inj.onStatus(T, AgentStatus.WORKING); // confirmed turn
|
||||
for (int i = 0; i < 10; i++) inj.onStatus(T, AgentStatus.UNKNOWN); // brief glitch, well under grace
|
||||
inj.onStatus(T, AgentStatus.IDLE); // working → idle: the real completion
|
||||
|
||||
assertEquals(List.of(), cap.failed, "a short unknown blip must not fail the turn");
|
||||
assertEquals(List.of(T), cap.completed, "the streak reset, so the turn still completes");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sendFailureDropsMessageAndFailsItsFuture() {
|
||||
FakeHerdr failing = new FakeHerdr().agentSendFailsWith("send_failed");
|
||||
|
||||
@@ -11,6 +11,7 @@ import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
@@ -71,4 +72,20 @@ class MessageServiceTest {
|
||||
assertEquals(MessageService.Outcome.REPLIED, reply.outcome());
|
||||
assertEquals("LGTM ship it", reply.text());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aWedgedWorkerResolvesTheSendAsFailedWithTheErrorContext() throws Exception {
|
||||
herdr.readText("API Error: Unable to connect to API (ENOTFOUND)");
|
||||
CompletableFuture<MessageService.Reply> send = sendAsync();
|
||||
awaitWaiting();
|
||||
|
||||
injector.onStatus(T, AgentStatus.IDLE); // deliver
|
||||
injector.onStatus(T, AgentStatus.WORKING); // worker starts the turn
|
||||
for (int i = 0; i < 130; i++) injector.onStatus(T, AgentStatus.UNKNOWN); // then wedges (CB-109)
|
||||
|
||||
MessageService.Reply reply = send.get(5, TimeUnit.SECONDS);
|
||||
assertEquals(MessageService.Outcome.WORKER_FAILED, reply.outcome());
|
||||
assertFalse(reply.completed(), "a wedge is terminal but not a successful completion");
|
||||
assertTrue(reply.text().contains("ENOTFOUND"), "the error screen is carried as the failure reason");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user