fleetd #796: document bootstrap retry readiness bound

fleetd/src/main/java/dev/ltms/fleet/config/FleetConfig.java

@@ -1466,7 +1466,7 @@ public record FleetConfig(
      *                     CALLING lead's own turn to end (its pane to report {@code IDLE} or
      *                     {@code DONE}) before ending that pane's process at all. See the
      *                     paragraph above.
-     * @param relaunchReadySeconds default 45 — bound on EACH of two separate waits that run after
+     * @param relaunchReadySeconds default 45 — bound on EACH of three separate waits that run after
      *                     the old lead's pane has been torn down and a fresh one launched: first,
      *                     for the fresh pane itself to reach a real turn boundary ({@code IDLE} or
      *                     {@code DONE}, never merely {@code BLOCKED}) — the safety gate, since
@@ -1479,8 +1479,9 @@ public record FleetConfig(
      *                     10s live), so a budget has to clear more than one scan interval to leave
      *                     any real margin for the CLI's own boot time; 20 was rejected for exactly
      *                     that reason — at a 10s scan interval it only buys two scans. 45 buys
-     *                     roughly four. Only a timeout on the FIRST wait (the pane never becomes
-     *                     ready) withholds {@code bootstrapText}.
+     *                     roughly four; third, to retry sending {@code bootstrapText} while herdr
+     *                     reports {@code agent_not_ready}. A timeout on the first wait or the third
+     *                     one withholds {@code bootstrapText}.
      * @param bootstrapText default a sentence naming the RESOLVED handover path — sent to the
      *                     fresh lead's pane once it reaches a real turn boundary after relaunch,
      *                     telling the fresh session where to read the handover and carry on. Left

fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java

@@ -232,7 +232,7 @@ public final class LeadRollover {
 
     /**
      * What is known about one token, right now — the answer {@link #status} gives. Distinguishes
-     * five terminal outcomes an approved roll can finish with, one in-flight outcome for a roll
+     * terminal outcomes an approved roll can finish with, one in-flight outcome for a roll
      * that has been approved but has not finished yet, and two answers for a token that names no
      * active work at all: still pending confirmation, or nothing known about this token at all.
      */
@@ -256,7 +256,8 @@ public final class LeadRollover {
          * that is, in fact, actively running. This is not sticky: the deferred continuation
          * overwrites this same entry with a terminal state ({@link #ROLLED}, {@link
          * #TURN_NEVER_SETTLED}, {@link #OLD_PANE_NEVER_DIED}, {@link #RELAUNCH_FAILED}, {@link
-         * #RELAUNCH_NEVER_READY}, {@link #RELAUNCH_NOT_RECOGNISED}, or {@link #FAILED}) once it
+         * #RELAUNCH_NEVER_READY}, {@link #RELAUNCH_NOT_RECOGNISED}, {@link #BOOTSTRAP_NEVER_SENT},
+         * or {@link #FAILED}) once it
          * finishes — including by throwing, which {@link #runRollover}'s catch turns into {@link
          * #FAILED} instead of leaving this entry stuck forever.
          */
@@ -305,6 +306,12 @@ public final class LeadRollover {
          * an operator should check why the tab was not recognised.
          */
         RELAUNCH_NOT_RECOGNISED,
+        /**
+         * A fresh lead was ready, but herdr kept reporting {@code agent_not_ready} while this class
+         * retried {@code bootstrapText} for {@code relaunchReadySeconds}. The fresh session did not
+         * receive its handover instruction.
+         */
+        BOOTSTRAP_NEVER_SENT,
         /**
          * The deferred continuation threw a {@link RuntimeException} and the continuation thread
          * died with it. Without this state, that throw would leave {@link #outcomes} holding {@link
@@ -731,7 +738,19 @@ public final class LeadRollover {
 
         IdentityResult identityResult = waitUntilRecognisedAsLead(newAgent.terminalId(),
                 cfg.relaunchReadySeconds());
-        agents.send(newAgent.terminalId(), cfg.bootstrapTextFor(p.handoverPath()));
+        BootstrapResult bootstrapResult = sendBootstrapWithRetry(newAgent.terminalId(),
+                cfg.bootstrapTextFor(p.handoverPath()), cfg.relaunchReadySeconds());
+        if (!bootstrapResult.sent()) {
+            log.warn("lead-rollover: bootstrapText was never sent to fresh terminal {} for lead '{}' "
+                            + "after agent_not_ready persisted for {}ms (token={}, configured={}s)",
+                    newAgent.terminalId(), leadName, bootstrapResult.elapsedMillis(), p.token(),
+                    cfg.relaunchReadySeconds());
+            outcomes.put(p.token(), new RollStatus(RollState.BOOTSTRAP_NEVER_SENT,
+                    "fresh terminal " + newAgent.terminalId() + " kept rejecting bootstrapText with "
+                            + "agent_not_ready for relaunchReadySeconds=" + cfg.relaunchReadySeconds()
+                            + "s (measured elapsed=" + bootstrapResult.elapsedMillis() + "ms)"));
+            return;
+        }
         if (!identityResult.ready()) {
             log.warn("lead-rollover: fresh terminal {} for lead '{}' is alive and bootstrapped, but "
                             + "was never recognised as a live lead — an operator should check why "
@@ -755,6 +774,30 @@ public final class LeadRollover {
                         + newAgent.terminalId()));
     }
 
+    /**
+     * Sends {@code bootstrapText}, retrying only the transient herdr {@code agent_not_ready} refusal
+     * until {@code readySeconds} elapses. All other failures propagate to {@link #runRollover}.
+     */
+    private BootstrapResult sendBootstrapWithRetry(String terminal, String bootstrapText, int readySeconds) {
+        long startMillis = nowMillis.getAsLong();
+        long deadline = startMillis + TimeUnit.SECONDS.toMillis(readySeconds);
+        while (nowMillis.getAsLong() < deadline) {
+            try {
+                agents.send(terminal, bootstrapText);
+                return new BootstrapResult(true, nowMillis.getAsLong() - startMillis);
+            } catch (HerdrException e) {
+                if (!"agent_not_ready".equals(e.code())) {
+                    throw e;
+                }
+                pollSleeper.run();
+            }
+        }
+        return new BootstrapResult(false, nowMillis.getAsLong() - startMillis);
+    }
+
+    /** The result of {@link #sendBootstrapWithRetry}. */
+    private record BootstrapResult(boolean sent, long elapsedMillis) {}
+
     /** Attempts {@link #captureAgentWithRetry} makes before letting the failure propagate. */
     static final int CAPTURE_RETRIES = 3;
 

fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java

@@ -2,10 +2,12 @@ package dev.ltms.fleet.lead;
 
 import ch.qos.logback.classic.Level;
 import ch.qos.logback.classic.spi.ILoggingEvent;
+import com.fasterxml.jackson.databind.JsonNode;
 import dev.ltms.fleet.config.FleetConfig;
 import dev.ltms.fleet.herdr.AgentControl;
 import dev.ltms.fleet.herdr.FakeHerdr;
 import dev.ltms.fleet.herdr.HerdrClient;
+import dev.ltms.fleet.herdr.HerdrException;
 import dev.ltms.fleet.herdr.WorkspaceControl;
 import dev.ltms.fleet.testing.CapturedLog;
 import org.junit.jupiter.api.DisplayName;
@@ -20,6 +22,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -1323,6 +1326,66 @@ class LeadRolloverTest {
                 + "so an operator reading status() has something to act on: " + status.detail());
     }
 
+    @Test
+    @DisplayName("[BOOTSTRAP 1] one agent_not_ready bootstrap refusal is retried and sends the "
+            + "handover instruction exactly once")
+    void transientAgentNotReadyRetriesBootstrapAndRolls() throws IOException {
+        FakeHerdr fake = herdrReadyForAFullRoll();
+        AtomicInteger refusedSends = new AtomicInteger();
+        HerdrClient transientRefusal = new HerdrClient() {
+            @Override
+            public JsonNode call(String method, Object params) {
+                if ("agent.prompt".equals(method) && refusedSends.getAndIncrement() == 0) {
+                    throw new HerdrException("herdr error [agent_not_ready]: agent.prompt failed",
+                            "agent_not_ready", null);
+                }
+                return fake.call(method, params);
+            }
+            @Override
+            public void close() {
+            }
+        };
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        AgentControl agents = new AgentControl(transientRefusal);
+        WorkspaceControl spaces = new WorkspaceControl(transientRefusal);
+        LeadRollover rollover = new LeadRollover(agents, spaces,
+                new LeadLauncher(agents, spaces, fleetConfigWithRelaunchableLead()),
+                () -> cfg(handover.toString()), _ -> null, _ -> LEAD_NAME,
+                () -> Map.of(NEW_TERMINAL, LEAD_NAME), fixedClock(clock), () -> { }, Runnable::run);
+
+        LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
+        assertTrue(rollover.confirm(LEAD, pending.token(), true).accepted());
+
+        assertEquals(2, refusedSends.get(), "one agent_not_ready refusal must be followed by one retry");
+        assertEquals(1, promptCallCount(fake), "only the successful retry reaches herdr delivery");
+        assertEquals(LeadRollover.RollState.ROLLED, rollover.status(pending.token()).state());
+    }
+
+    @Test
+    @DisplayName("[BOOTSTRAP 2] persistent agent_not_ready records BOOTSTRAP_NEVER_SENT and "
+            + "releases the single-flight claim")
+    void persistentAgentNotReadyRecordsBootstrapNeverSentAndReleasesClaim() throws IOException {
+        FakeHerdr fake = herdrReadyForAFullRoll();
+        fake.agentSendFailsWith("agent_not_ready");
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        LeadRollover rollover = newRolloverForAFullRoll(fake, cfg(handover.toString()),
+                () -> clock.addAndGet(1_000), Map.of(NEW_TERMINAL, LEAD_NAME));
+
+        LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
+        assertTrue(rollover.confirm(LEAD, pending.token(), true).accepted());
+
+        LeadRollover.RollStatus status = rollover.status(pending.token());
+        assertEquals(LeadRollover.RollState.BOOTSTRAP_NEVER_SENT, status.state());
+        assertNotEquals(LeadRollover.RollState.ROLLED, status.state());
+        assertNotEquals(LeadRollover.RollState.FAILED, status.state());
+
+        LeadRollover.PendingRollover retry = rollover.open(LEAD, "retry after bootstrap timeout");
+        assertTrue(rollover.confirm(LEAD, retry.token(), true).accepted(),
+                "the terminal outcome must release the single-flight claim");
+    }
+
     // ---- fleetd #726 unit 3: confirm() single-flights one roll at a time per lead ---------------
 
     @Test