fleet/fleetd
3
0
Fork 0
Code Issues 73 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fleetd #489: nudge the /clear submit keystroke before bootstrapText #490

Merged
ltms merged 2 commits from worker/489-001902-2 into main 2026-09-12 02:52:52 +02:00
Conversation 1 Commits 2 Files Changed 2
+306 -20
2 changed files with 306 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java
+128 -20
View File
@@ -44,15 +44,18 @@ import java.util.function.Supplier;
* {@code continuationRunner} before returning. That continuation is what actually touches the pane,
* once the calling turn has ended, in this order:
* <ol>
* <li>wait for the lead's own pane to report an injectable state — i.e. wait for the very
* {@code confirm()} call that approved this roll to finish its turn — bounded by
* {@code turnSettleSeconds}. <strong>If this never happens, nothing else in this list runs:
* no {@code /clear} is ever sent.</strong> A lead that never goes idle is a lead still doing
* real work, and clearing it would throw away live context — exactly the failure this
* correction exists to prevent.</li>
* <li>wait for the lead's own pane to report a real turn boundary — {@code IDLE} or {@code
* DONE}, never merely {@code BLOCKED} — i.e. wait for the very {@code confirm()} call that
* approved this roll to finish its turn — bounded by {@code turnSettleSeconds}. <strong>If
* this never happens, nothing else in this list runs: no {@code /clear} is ever sent.</strong>
* A lead that never goes idle is a lead still doing real work, and clearing it would throw
* away live context — exactly the failure this correction exists to prevent.</li>
* <li>{@code agents.send(lead, "/clear")}</li>
* <li>wait again for the pane to report injectable, bounded by {@code clearSettleSeconds} (this
* is the original, pre-correction wait — still here, just no longer the only one)</li>
* <li>wait for {@code /clear} to be picked up and settle, bounded by {@code clearSettleSeconds}
* (fleetd #489: no longer a plain re-check of the same boundary — {@code /clear} starts no
* turn of its own, so this instead nudges the submit keystroke while no pickup has been seen,
* then waits for a real {@code WORKING} → {@code IDLE}/{@code DONE} boundary once one has;
* see {@link #waitForClearPickupAndSettle})</li>
* <li>{@code agents.send(lead, cfg.bootstrapTextFor(p.handoverPath()))}</li>
* </ol>
* A {@link #confirm} that returns {@link RollDecision#approved()} therefore means <em>"every gate
@@ -111,6 +114,17 @@ public final class LeadRollover {
/** Poll interval while waiting for the lead's pane to settle after {@code /clear}. */
static final long SETTLE_POLL_MS = 250;
/**
* How many consecutive not-yet-picked-up polls {@link #waitForClearPickupAndSettle} allows
* before releasing rather than wedging the roll — the same constant and the same
* release-not-wedge choice {@link dev.ltms.fleet.inject.Injector} already makes for its own
* post-turn {@code /clear} housekeeping (fleetd #306). <strong>This bounds the number of
* consecutive polls, not the number of nudges:</strong> the first {@code PICKUP_GRACE_POLLS - 1}
* of those polls each send a nudge, and the {@code PICKUP_GRACE_POLLS}th releases instead of
* nudging again — so 8 polls produce 7 nudges, not 8.
*/
static final int PICKUP_GRACE_POLLS = 8;
/**
* One request opened by {@link #open}, pending its {@link #confirm} (or {@link #cancel}).
*
@@ -381,7 +395,7 @@ public final class LeadRollover {
// /clear is housekeeping, not a delegated turn, and routing it through Injector wedges the
// pane forever (see this class's javadoc).
agents.send(lead, "/clear");
boolean clearSettled = waitUntilAtTurnBoundary(lead, cfg.clearSettleSeconds());
boolean clearSettled = waitForClearPickupAndSettle(lead, cfg.clearSettleSeconds());
if (!clearSettled) {
log.warn("lead-rollover: pane {} did not reach a turn boundary (IDLE or DONE) within {}s "
+ "after /clear — NOT sending bootstrapText (token={})",
@@ -440,11 +454,14 @@ public final class LeadRollover {
/**
* Poll {@link AgentControl#status} until {@code target} reports a real turn boundary — {@link
* AgentStatus#IDLE} or {@link AgentStatus#DONE} — bounded by {@code settleSeconds}. Used twice
* by {@link #runRollover}: once to wait for the CALLING turn's own pane to settle (the {@code
* turnSettleSeconds} gate that makes this correction safe), and once to wait for the pane to
* re-settle after {@code /clear}. A failed status read degrades to "not yet settled" and is
* retried on the next poll, the same posture {@code LeadHeartbeatLoop} and {@code
* AgentStatus#IDLE} or {@link AgentStatus#DONE} — bounded by {@code settleSeconds}. Used once by
* {@link #runRollover}, to wait for the CALLING turn's own pane to settle before {@code /clear}
* is ever sent at all — the {@code turnSettleSeconds} gate that makes this correction safe. The
* SECOND wait, after {@code /clear}, is {@link #waitForClearPickupAndSettle} instead (fleetd
* #489) — a plain boundary check is not enough there, because {@code /clear} starts no turn of
* its own, so this method would (wrongly) report "settled" on its very first poll whether or not
* {@code /clear} was actually picked up. A failed status read degrades to "not yet settled" and
* is retried on the next poll, the same posture {@code LeadHeartbeatLoop} and {@code
* HerdrPeerLauncher}'s readiness gate already take toward an unreadable status.
*
* <p><strong>Deliberately not {@link AgentStatus#injectable()}.</strong> {@code injectable()}
@@ -452,12 +469,12 @@ public final class LeadRollover {
* turn" — and it accepts {@link AgentStatus#BLOCKED} for that purpose, because a pane paused on
* an approval prompt is safe to queue a message behind. This class asks a stricter question —
* "has the turn actually ended" — and {@code BLOCKED} answers no: it is a live turn that is
* merely paused, not one that has finished. Reusing {@code injectable()} here would let both
* waits fire into an open approval prompt mid-turn (the first wait would send {@code /clear}
* while the lead's own {@code confirm()}-calling turn is still live and paused on a prompt; the
* second would send {@code bootstrapText} the same way after {@code /clear}) — exactly the
* live-context-destroying failure the {@code turnSettleSeconds} gate exists to prevent. Do not
* "simplify" this back to {@code injectable()}.
* merely paused, not one that has finished. Reusing {@code injectable()} here would let this
* wait fire {@code /clear} while the lead's own {@code confirm()}-calling turn is still live and
* paused on a prompt — exactly the live-context-destroying failure the {@code turnSettleSeconds}
* gate exists to prevent. Do not "simplify" this back to {@code injectable()}. ({@link
* #waitForClearPickupAndSettle} keeps the same exclusion of {@code BLOCKED}, for the same
* reason, on the second wait.)
*/
private boolean waitUntilAtTurnBoundary(String target, int settleSeconds) {
long deadline = nowMillis.getAsLong() + TimeUnit.SECONDS.toMillis(settleSeconds);
@@ -477,4 +494,95 @@ public final class LeadRollover {
}
return false;
}
/**
* The SECOND wait in {@link #runRollover} — after {@code /clear} has been sent, waits for it to
* settle, bounded by {@code settleSeconds}. <strong>fleetd #489 — the paste-race fix.</strong>
* {@code /clear} does not start a real turn of its own, so a pane with no submit race simply
* stays {@link AgentStatus#IDLE} the whole time: {@link #waitUntilAtTurnBoundary} would (wrongly)
* call that "settled" on its very first poll, whether or not the {@code /clear} Enter actually
* landed. That was Fault 1, measured live on 2026-09-12 — the second gate was a no-op, so a
* {@code bootstrapText} send followed immediately, racing Fault 2: {@link AgentControl#submit}'s
* own javadoc already records that the submit accompanying a delivery "can race the paste —
* especially right as the worker's TUI becomes interactive — leaving the text unsubmitted"
* (CB-113). Because {@code runRollover} deliberately bypasses {@code Injector} for {@code
* /clear} (see this class's javadoc), it inherited none of {@code Injector}'s nudging — so the
* lost {@code /clear} Enter sat in the input box and {@code bootstrapText} was typed right after
* it, landing as one concatenated line.
*
* <p>This method copies the pickup-nudge pattern {@link dev.ltms.fleet.inject.Injector} already
* ships for exactly this, on its own post-turn {@code /clear} housekeeping (fleetd #306; see
* {@code Injector.java:288-340} and {@code Injector.java:437-442}):
* <ul>
* <li>an {@link AgentStatus#WORKING} sample means {@code /clear} was picked up as a real
* turn;</li>
* <li>until that happens, each poll that still reports {@link AgentStatus#IDLE} or {@link
* AgentStatus#DONE} re-sends the submit keystroke ({@link AgentControl#submit}) to nudge
* the raced Enter — for the first {@code PICKUP_GRACE_POLLS - 1} of {@link
* #PICKUP_GRACE_POLLS} consecutive such polls (i.e. {@code PICKUP_GRACE_POLLS - 1}
* nudges: 7, not 8, given {@code PICKUP_GRACE_POLLS = 8}). A second Enter on an empty
* Claude Code prompt is a no-op, so repeating it is safe;</li>
* <li>the {@code PICKUP_GRACE_POLLS}th consecutive such poll, with {@code WORKING} still never
* observed, releases rather than wedges the roll instead of nudging again — the same
* choice {@code Injector} makes — and returns {@code true} anyway, logged at {@code info}
* so an operator can see which path ran;</li>
* <li>once {@code WORKING} has been observed, nudging stops and this instead waits for a real
* {@code working → IDLE/DONE} completion boundary before returning {@code true}.</li>
* </ul>
*
* <p><strong>{@link AgentStatus#BLOCKED} is deliberately excluded from both the nudge and the
* boundary check</strong> — the same reasoning as {@link #waitUntilAtTurnBoundary}'s own
* javadoc: a paused live turn is not a settled one, and re-sending Enter into an open approval
* prompt could wrongly answer it. A {@code BLOCKED} sample (or an unreadable/{@link
* AgentStatus#UNKNOWN} one) simply keeps this polling, with no nudge and no release, until either
* a real boundary is reached or {@code settleSeconds} runs out.
*
* <p>{@link AgentControl#submit} can itself throw; a {@link RuntimeException} from it is
* swallowed and logged at {@code debug}, exactly like {@code Injector.java:437-442} — a failed
* nudge must not abort the roll.
*
* @return {@code true} once {@code /clear} has settled, or once the nudge budget was exhausted
* with no pickup ever observed (released rather than wedged); {@code false} if {@code
* settleSeconds} elapses first — the caller must NOT send {@code bootstrapText} in that
* case, exactly as before this fix
*/
private boolean waitForClearPickupAndSettle(String target, int settleSeconds) {
long deadline = nowMillis.getAsLong() + TimeUnit.SECONDS.toMillis(settleSeconds);
boolean pickedUp = false; // a WORKING sample has been observed since /clear was sent
int idlePollsAwaitingPickup = 0;
while (nowMillis.getAsLong() < deadline) {
AgentStatus status;
try {
status = agents.status(target);
} catch (RuntimeException e) {
log.debug("lead-rollover: status check failed while waiting for {} to settle after "
+ "/clear: {}", target, e.toString());
status = null;
}
if (status == AgentStatus.WORKING) {
pickedUp = true;
} else if (status == AgentStatus.IDLE || status == AgentStatus.DONE) {
if (pickedUp) {
return true; // a real WORKING -> IDLE/DONE completion boundary
}
if (++idlePollsAwaitingPickup >= PICKUP_GRACE_POLLS) {
log.info("lead-rollover: /clear on {} was never observed as WORKING after {} "
+ "consecutive IDLE/DONE polls ({} of those were nudged) — "
+ "releasing rather than wedging the roll",
target, PICKUP_GRACE_POLLS, PICKUP_GRACE_POLLS - 1);
return true;
}
try {
agents.submit(target); // nudge a raced Enter (CB-113) so /clear actually submits
} catch (RuntimeException e) {
log.debug("lead-rollover: resubmit to {} failed (will retry next poll): {}",
target, e.getMessage());
}
}
// AgentStatus.BLOCKED or UNKNOWN (or an unreadable status, above): neither a pickup
// signal nor a boundary — keep polling without nudging or releasing.
settleSleeper.run();
}
return false;
}
}
fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java
+178
View File
@@ -458,6 +458,184 @@ class LeadRolloverTest {
assertEquals(0, bootstrapSends, "bootstrapText must never be sent when /clear did not settle");
}
// ---- fleetd #489: the second wait nudges the /clear pickup instead of being a no-op --------
/** Every {@code agent.send_keys} call {@code herdr} recorded — the submit-keystroke nudge. */
private static long sendKeysCallCount(FakeHerdr herdr) {
return herdr.calls.stream().filter(c -> "agent.send_keys".equals(c.method())).count();
}
@Test
@DisplayName("[fleetd #489] a pane that stays IDLE the whole time (the paste-race case, "
+ "measured live 2026-09-12) is nudged between /clear and bootstrapText, never lets "
+ "them concatenate into one line")
void clearPickupIsNudgedBeforeBootstrapTextWhenPaneStaysIdle() throws IOException {
FakeHerdr herdr = new FakeHerdr(); // default agentStatus is "idle" throughout — no WORKING sample ever
Path handover = writeHandover("handover contents");
AtomicLong clock = new AtomicLong(1_000);
LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), fixedClock(clock));
LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail());
int clearIdx = -1;
int bootstrapIdx = -1;
int firstNudgeIdx = -1;
for (int i = 0; i < herdr.calls.size(); i++) {
FakeHerdr.Call c = herdr.calls.get(i);
if ("agent.prompt".equals(c.method()) && String.valueOf(c.params()).contains("/clear") && clearIdx < 0) {
clearIdx = i;
} else if ("agent.prompt".equals(c.method()) && String.valueOf(c.params()).contains("read the handover file")) {
bootstrapIdx = i;
} else if ("agent.send_keys".equals(c.method()) && firstNudgeIdx < 0) {
firstNudgeIdx = i;
}
}
assertTrue(clearIdx >= 0, "/clear must have been sent");
assertTrue(bootstrapIdx >= 0, "bootstrapText must have been sent");
assertTrue(firstNudgeIdx >= 0, "at least one agent.send_keys nudge must go out — the pane "
+ "never reported WORKING, so the /clear Enter may have raced the paste, and only a "
+ "re-sent Enter proves the clear rather than concatenating bootstrapText onto "
+ "whatever sits unsubmitted in the input box");
assertTrue(firstNudgeIdx > clearIdx, "the nudge must happen AFTER /clear was sent, got call "
+ "order: " + herdr.calls);
assertTrue(firstNudgeIdx < bootstrapIdx, "the nudge must happen BEFORE bootstrapText is "
+ "sent — never concatenated onto the same input line, got call order: " + herdr.calls);
}
@Test
@DisplayName("[fleetd #489] once a WORKING sample confirms /clear was picked up, nudging stops "
+ "and bootstrapText is still sent after the pane returns to IDLE")
void pickupSeenStopsNudgingAndBootstrapTextIsSent() throws IOException {
FakeHerdr fake = new FakeHerdr();
// Scripts the SECOND wait only: idle (default) until /clear is sent, then the first status
// poll after /clear reports WORKING (a confirmed pickup), and every poll after that reports
// IDLE (the completion boundary). The first wait (turnSettleSeconds) never sees this
// sequence — it passes on its own first poll, before /clear is ever sent, on the default
// "idle" status.
AtomicLong postClearGetCalls = new AtomicLong(0);
HerdrClient scriptsPickupThenIdle = new HerdrClient() {
private volatile boolean clearSent = false;
@Override
public JsonNode call(String method, Object params) throws HerdrException {
if ("agent.prompt".equals(method) && String.valueOf(params).contains("/clear")) {
clearSent = true;
}
if (clearSent && "agent.get".equals(method)) {
long n = postClearGetCalls.incrementAndGet();
fake.agentStatus(n == 1 ? "working" : "idle");
}
return fake.call(method, params);
}
@Override
public void close() {
fake.close();
}
};
Path handover = writeHandover("handover contents");
AtomicLong clock = new AtomicLong(1_000);
LeadRollover rollover = newRollover(scriptsPickupThenIdle, cfg(handover.toString()), fixedClock(clock));
LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail());
assertEquals(2, promptCallCount(fake), "a confirmed WORKING pickup followed by IDLE must "
+ "still complete the full roll — /clear then bootstrapText");
assertEquals(0, sendKeysCallCount(fake), "once WORKING was observed, nudging must stop "
+ "immediately — no agent.send_keys call should ever have been needed or sent");
assertTrue(postClearGetCalls.get() >= 2, "the pane's status must have been polled AGAIN "
+ "after the WORKING sample, before bootstrapText was sent — this is what proves "
+ "the method actually waited for the WORKING -> IDLE completion boundary instead "
+ "of returning as soon as pickup was seen (or worse, without polling at all, as a "
+ "stub that just returns true would); got " + postClearGetCalls.get()
+ " agent.get call(s) after /clear");
}
@Test
@DisplayName("[fleetd #489] a pane that never reaches a turn boundary after /clear (stuck at "
+ "UNKNOWN, never WORKING either) lets clearSettleSeconds expire — bootstrapText is "
+ "never sent")
void clearPickupNeverSettlesWhenStatusNeverReachesABoundary() throws IOException {
FakeHerdr fake = new FakeHerdr();
HerdrClient stuckUnknownAfterClear = new HerdrClient() {
@Override
public JsonNode call(String method, Object params) throws HerdrException {
if ("agent.prompt".equals(method) && String.valueOf(params).contains("/clear")) {
// "wedged" maps to AgentStatus.UNKNOWN (see AgentStatus#fromWire) — neither a
// pickup signal (WORKING) nor a boundary (IDLE/DONE), and distinct from the
// already-covered BLOCKED case below.
fake.agentStatus("wedged");
}
return fake.call(method, params);
}
@Override
public void close() {
fake.close();
}
};
Path handover = writeHandover("handover contents");
FleetConfig.LeadRollover config =
new FleetConfig.LeadRollover(handover.toString(), true, 3600, 20, 1 /*clearSettleSeconds*/, "boot text");
AtomicLong clock = new AtomicLong(1_000);
LeadRollover rollover = newRollover(stuckUnknownAfterClear, config, () -> clock.addAndGet(500));
LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
assertTrue(decision.accepted(), "every synchronous gate passes; the refusal is logged only, "
+ "deep inside the deferred continuation");
assertEquals(1, promptCallCount(fake), "exactly one agent.prompt call — the /clear — and "
+ "nothing else");
long bootstrapSends = fake.calls.stream()
.filter(c -> "agent.prompt".equals(c.method()))
.filter(c -> String.valueOf(c.params()).contains("boot text"))
.count();
assertEquals(0, bootstrapSends, "bootstrapText must never be sent when the pane never "
+ "reaches a turn boundary after /clear, whether WORKING was ever observed or not");
assertEquals(0, sendKeysCallCount(fake), "an UNKNOWN status is neither a pickup signal nor "
+ "a boundary — it must never be nudged");
}
@Test
@DisplayName("[fleetd #489] a submit() nudge that throws does not abort the roll — /clear and "
+ "bootstrapText are both still sent")
void submitThatThrowsDoesNotAbortTheRoll() throws IOException {
FakeHerdr fake = new FakeHerdr(); // default idle throughout — nudging will be attempted
HerdrClient throwsOnSubmit = new HerdrClient() {
@Override
public JsonNode call(String method, Object params) throws HerdrException {
if ("agent.send_keys".equals(method)) {
throw new RuntimeException("simulated herdr transport failure on submit");
}
return fake.call(method, params);
}
@Override
public void close() {
fake.close();
}
};
Path handover = writeHandover("handover contents");
AtomicLong clock = new AtomicLong(1_000);
LeadRollover rollover = newRollover(throwsOnSubmit, cfg(handover.toString()), fixedClock(clock));
LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail());
var prompts = fake.calls.stream().filter(c -> "agent.prompt".equals(c.method())).toList();
assertEquals(2, prompts.size(), "a throwing submit() must be swallowed, not abort the roll "
+ "— /clear and bootstrapText must both still be sent");
assertTrue(prompts.get(0).params().toString().contains("/clear"));
assertTrue(prompts.get(1).params().toString().contains("read the handover file"));
}
@Test
@DisplayName("a full successful roll sends /clear then bootstrapText, in order, and consumes the token")
void successfulRollSendsClearThenBootstrapTextAndConsumesTheToken() throws IOException {