fleetd #726 unit 3 fix: release the single-flight claim when continuationRunner rejects the hand-off

confirm() takes the per-lead-terminal claim before handing the roll to
continuationRunner, and the only release path was runRollover's own
finally. If continuationRunner.accept itself throws, runRollover never
starts, so that finally never runs, and nothing else ever writes
rollingByTerminal — the claim is held forever and the terminal can never
be rolled again. This differs from fleetd #615, which covers a throw
INSIDE the continuation (runRollover already catches that and still
releases the claim) — this is a throw from the hand-off itself, which
is not reachable with today's virtual-thread runner but would be with
a bounded executor's RejectedExecutionException.

confirm() now catches that throw, releases the claim, and overwrites
the IN_PROGRESS outcome with a terminal FAILED one, matching how a
throw inside the continuation is already surfaced.

fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java

@@ -164,7 +164,13 @@ public final class LeadRollover {
          * The handover file's modified time is not after {@link #open}'s request timestamp, or is
          * older than {@code maxDocAgeSeconds}.
          */
-        HANDOVER_STALE
+        HANDOVER_STALE,
+        /**
+         * This lead terminal already has a roll running: an earlier {@link #confirm} call claimed
+         * it and that roll's continuation has not released it yet. {@code detail} names the lead
+         * terminal and the token that holds the claim.
+         */
+        ROLL_ALREADY_RUNNING
     }
 
     /**
@@ -305,6 +311,15 @@ public final class LeadRollover {
      */
     private final Consumer<Runnable> continuationRunner;
     private final Map<String, PendingRollover> pending = new ConcurrentHashMap<>();
+    /**
+     * Lead terminal → the token of the roll currently holding that terminal exclusive, for
+     * {@link #confirm}'s single-flight claim. {@link #confirm} claims an entry here with an
+     * atomic put-if-absent once every other gate has passed, refusing with {@link
+     * RefusalReason#ROLL_ALREADY_RUNNING} when a claim is already held; {@link #runRollover}
+     * releases it in a {@code finally}, on both the success and the thrown-exception path. A
+     * terminal absent from this map has no roll currently in flight for it.
+     */
+    private final Map<String, String> rollingByTerminal = new ConcurrentHashMap<>();
     /**
      * Finished tokens → what actually happened, for {@link #status}. Bounded by {@link
      * #OUTCOME_HISTORY_CAP}, oldest evicted first ({@code removeEldestEntry} on an insertion-order
@@ -487,6 +502,16 @@ public final class LeadRollover {
             return docCheck;
         }
 
+        // Single-flight claim: atomic put-if-absent, taken only after every other gate has
+        // passed, so a refused confirm() never takes it. A non-null previous value means a
+        // different, still-running roll already holds this lead terminal.
+        String holder = rollingByTerminal.putIfAbsent(p.leadTerminal(), token);
+        if (holder != null) {
+            return RollDecision.refused(RefusalReason.ROLL_ALREADY_RUNNING,
+                    "lead terminal " + p.leadTerminal() + " already has a roll running under token "
+                            + holder);
+        }
+
         // Record IN_PROGRESS BEFORE removing from `pending` — see RollState#IN_PROGRESS and
         // OUTCOME_HISTORY_CAP's javadoc. This ordering means `token` is written into `outcomes`
         // while it is STILL present in `pending`; status() checks `outcomes` first (see that
@@ -499,7 +524,23 @@ public final class LeadRollover {
         pending.remove(token);
         log.info("lead-rollover: confirmed token={} lead={} — roll scheduled once the calling turn ends",
                 token, callerTerminal);
-        continuationRunner.accept(() -> runRollover(p, cfg));
+        try {
+            continuationRunner.accept(() -> runRollover(p, cfg));
+        } catch (RuntimeException e) {
+            // continuationRunner can reject the hand-off itself (e.g. a bounded executor's
+            // RejectedExecutionException) before runRollover ever starts, so runRollover's own
+            // finally — the only other place that releases rollingByTerminal — never runs either.
+            // Release the claim here and overwrite the IN_PROGRESS entry with a terminal outcome,
+            // or this lead terminal could never be rolled again and status() would report
+            // IN_PROGRESS forever for a roll that in fact never started.
+            log.warn("lead-rollover: continuationRunner rejected token={} lead={}: {} — the roll "
+                            + "never started; releasing its claim and reporting it as FAILED",
+                    token, callerTerminal, e.toString(), e);
+            rollingByTerminal.remove(p.leadTerminal(), token);
+            outcomes.put(token, new RollStatus(RollState.FAILED,
+                    "continuationRunner rejected this roll before it ever started: " + e.toString()
+                            + " — the roll never ran; open() a fresh rollover request"));
+        }
         return RollDecision.approved();
     }
 
@@ -539,6 +580,12 @@ public final class LeadRollover {
                     "the roll's continuation threw " + e.toString() + " — the roll is dead and will "
                             + "not retry itself; check the daemon log for the stack trace, then open() "
                             + "a fresh rollover request"));
+        } finally {
+            // Release the single-flight claim on both the normal return and the thrown-exception
+            // path above — a release only on success would leave this lead terminal unrollable
+            // forever after one failure. The conditional two-argument remove only clears the
+            // entry this roll itself holds, never a different roll's claim on the same terminal.
+            rollingByTerminal.remove(p.leadTerminal(), p.token());
         }
     }
 

fleetd/src/main/java/dev/ltms/fleet/msg/MessageService.java

@@ -355,6 +355,14 @@ public final class MessageService {
      */
     private final ConcurrentHashMap<String, Boolean> queuedDeliveries = new ConcurrentHashMap<>();
     private final AtomicLong ticketSeq = new AtomicLong();
+    /**
+     * Minted once per {@code MessageService} instance and folded into every ticket id (see
+     * {@link #sendAsync(String, String, Runnable, String)}). {@link #ticketSeq} alone restarts at
+     * zero for every instance, so without this a ticket id can be reused across instances and
+     * resolve to an unrelated {@link Task} with no error; this nonce makes that impossible, because
+     * an id minted by one instance can never match the id space of another.
+     */
+    private final String ticketBootNonce = UUID.randomUUID().toString().substring(0, 6);
     private final ExecutorService asyncExecutor = Executors.newThreadPerTaskExecutor(
             Thread.ofVirtual().name("bridge-async-", 0).factory());
 
@@ -1321,7 +1329,7 @@ public final class MessageService {
      * @return the ticket to poll for the eventual result
      */
     public String sendAsync(String target, String content, Runnable onAccepted, String creatorTerminal) {
-        String ticket = "task-" + ticketSeq.incrementAndGet();
+        String ticket = "task-" + ticketBootNonce + "-" + ticketSeq.incrementAndGet();
         Task task = new Task(ticket, target, nowNanos, creatorTerminal);
         tasks.put(ticket, task);
         if (pushLoop != null) {

fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java

@@ -1305,8 +1305,12 @@ class LeadRolloverTest {
         int rolls = LeadRollover.OUTCOME_HISTORY_CAP + 50;
         String[] tokens = new String[rolls];
         for (int i = 0; i < rolls; i++) {
-            LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full #" + i);
-            LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
+            // a distinct terminal per iteration: confirm() single-flights per lead terminal, so
+            // reusing LEAD here would refuse every roll after the first instead of building up
+            // OUTCOME_HISTORY_CAP+50 simultaneously in-flight entries.
+            String terminal = "term_cap_" + i;
+            LeadRollover.PendingRollover pending = rollover.open(terminal, "context is full #" + i);
+            LeadRollover.RollDecision decision = rollover.confirm(terminal, pending.token(), true);
             assertTrue(decision.accepted(), "roll #" + i + " should have been approved: " + decision.detail());
             tokens[i] = pending.token();
         }
@@ -1447,4 +1451,195 @@ class LeadRolloverTest {
         assertTrue(status.detail().contains("HerdrException"), "the detail must name the exception "
                 + "so an operator reading status() has something to act on: " + status.detail());
     }
+
+    // ---- fleetd #726 unit 3: confirm() single-flights per lead terminal -----------------------
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 1] a second confirm() for the SAME lead terminal is refused with "
+            + "ROLL_ALREADY_RUNNING while the first roll is still in flight, continuationRunner ran "
+            + "exactly once, and the claim is released once that first roll finishes")
+    void secondConfirmForSameTerminalIsRefusedWhileFirstRollIsStillInFlight() throws IOException {
+        FakeHerdr herdr = new FakeHerdr(); // default idle — the held roll WOULD complete once run
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        HoldingRunner runner = new HoldingRunner();
+        LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()),
+                fixedClock(clock), runner);
+
+        LeadRollover.PendingRollover first = rollover.open(LEAD, "context is full");
+        LeadRollover.RollDecision firstDecision = rollover.confirm(LEAD, first.token(), true);
+        assertTrue(firstDecision.accepted(), "expected approval; got: " + firstDecision.reason()
+                + " / " + firstDecision.detail());
+        assertEquals(1, runner.heldCount(), "sanity: the first roll is held, not run yet");
+
+        LeadRollover.PendingRollover second = rollover.open(LEAD, "a second request for the same terminal");
+        LeadRollover.RollDecision secondDecision = rollover.confirm(LEAD, second.token(), true);
+
+        assertFalse(secondDecision.accepted());
+        assertEquals(LeadRollover.RefusalReason.ROLL_ALREADY_RUNNING, secondDecision.reason());
+        assertTrue(secondDecision.detail().contains(LEAD), "the refusal detail must name the lead "
+                + "terminal: " + secondDecision.detail());
+        assertTrue(secondDecision.detail().contains(first.token()), "the refusal detail must name "
+                + "the token holding the claim: " + secondDecision.detail());
+        assertEquals(1, runner.heldCount(), "the refused confirm() must never have reached "
+                + "continuationRunner — it ran exactly once, for the first roll only");
+
+        runner.runNext(); // let the first (and only) held roll finish
+        assertEquals(2, promptCallCount(herdr), "continuationRunner ran exactly once: /clear then "
+                + "bootstrapText, for the first roll only");
+
+        // The claim must have been released once the first roll's continuation finished — a
+        // fresh request for the SAME terminal is now approved.
+        LeadRollover.PendingRollover third = rollover.open(LEAD, "retry after the first roll finished");
+        LeadRollover.RollDecision thirdDecision = rollover.confirm(LEAD, third.token(), true);
+        assertTrue(thirdDecision.accepted(), "the claim must have been released once the first "
+                + "roll finished: " + thirdDecision.reason() + " / " + thirdDecision.detail());
+    }
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 2] two DIFFERENT lead terminals confirm without either being "
+            + "refused, and the continuation runs once per terminal")
+    void differentLeadTerminalsConfirmIndependently() throws IOException {
+        FakeHerdr herdr = new FakeHerdr(); // default idle — both rolls complete
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), fixedClock(clock));
+
+        LeadRollover.PendingRollover a = rollover.open(LEAD, "context is full");
+        LeadRollover.PendingRollover b = rollover.open(OTHER_LEAD, "context is full too");
+
+        LeadRollover.RollDecision aDecision = rollover.confirm(LEAD, a.token(), true);
+        LeadRollover.RollDecision bDecision = rollover.confirm(OTHER_LEAD, b.token(), true);
+
+        assertTrue(aDecision.accepted(), "expected approval; got: " + aDecision.reason() + " / " + aDecision.detail());
+        assertTrue(bDecision.accepted(), "expected approval; got: " + bDecision.reason() + " / " + bDecision.detail());
+        assertEquals(4, promptCallCount(herdr), "both rolls ran to completion — /clear and "
+                + "bootstrapText, once per terminal");
+    }
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 3] a confirm() refused for OPERATOR_NOT_CONFIRMED does not take "
+            + "the single-flight claim — a later valid confirm on the same terminal is still approved")
+    void refusedConfirmDoesNotTakeTheClaim() throws IOException {
+        FakeHerdr herdr = new FakeHerdr();
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        LeadRollover rollover = newRollover(herdr, cfg(handover.toString(), true), fixedClock(clock));
+
+        LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
+        LeadRollover.RollDecision refused = rollover.confirm(LEAD, pending.token(), false);
+        assertFalse(refused.accepted());
+        assertEquals(LeadRollover.RefusalReason.OPERATOR_NOT_CONFIRMED, refused.reason());
+
+        // the same token stays pending after a gate refusal — retrying it with operator
+        // confirmation must succeed, proving the earlier refusal never took the claim.
+        LeadRollover.RollDecision approved = rollover.confirm(LEAD, pending.token(), true);
+        assertTrue(approved.accepted(), "a confirm() refused for an existing gate must never have "
+                + "taken the single-flight claim: " + approved.reason() + " / " + approved.detail());
+    }
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 4] a roll whose continuation throws a RuntimeException still "
+            + "releases the claim — a fresh confirm() on that terminal is approved afterwards")
+    void throwingContinuationStillReleasesTheClaim() throws IOException {
+        FakeHerdr fake = new FakeHerdr(); // default idle — the turn-settle wait passes immediately
+        HerdrClient throwsOnClear = new HerdrClient() {
+            @Override
+            public JsonNode call(String method, Object params) throws HerdrException {
+                if ("agent.prompt".equals(method) && String.valueOf(params).contains("/clear")) {
+                    throw new HerdrException("simulated herdr transport failure sending /clear");
+                }
+                return fake.call(method, params);
+            }
+
+            @Override
+            public void close() {
+                fake.close();
+            }
+        };
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        LeadRollover rollover = newRollover(throwsOnClear, cfg(handover.toString()), fixedClock(clock));
+
+        LeadRollover.PendingRollover first = rollover.open(LEAD, "context is full");
+        LeadRollover.RollDecision decision = rollover.confirm(LEAD, first.token(), true);
+        assertTrue(decision.accepted(), "every synchronous gate passes; the throw happens only "
+                + "inside the deferred continuation, which this test's synchronous runner has "
+                + "already run to completion by the time confirm() returns");
+
+        LeadRollover.RollStatus status = rollover.status(first.token());
+        assertEquals(LeadRollover.RollState.FAILED, status.state(), "sanity: the continuation threw "
+                + "and left a terminal FAILED outcome: " + status.detail());
+
+        LeadRollover.PendingRollover second = rollover.open(LEAD, "retry after the failure");
+        LeadRollover.RollDecision retryDecision = rollover.confirm(LEAD, second.token(), true);
+        assertTrue(retryDecision.accepted(), "the claim must have been released even though the "
+                + "continuation threw — a release only on the success path would leave this lead "
+                + "terminal unrollable forever after one failure: " + retryDecision.reason() + " / "
+                + retryDecision.detail());
+    }
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 5] cancel(token) after a successful confirm(token) returns false "
+            + "and does not release the claim held by that roll")
+    void cancelAfterConfirmDoesNotReleaseTheClaim() throws IOException {
+        FakeHerdr herdr = new FakeHerdr();
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        HoldingRunner runner = new HoldingRunner();
+        LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()),
+                fixedClock(clock), runner);
+
+        LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
+        LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
+        assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail());
+
+        assertFalse(rollover.cancel(pending.token()), "confirm() already removed the token from "
+                + "pending, so cancel() must find nothing for it");
+
+        LeadRollover.PendingRollover another = rollover.open(LEAD,
+                "a second request while the first roll is still held");
+        LeadRollover.RollDecision anotherDecision = rollover.confirm(LEAD, another.token(), true);
+        assertFalse(anotherDecision.accepted(), "cancel() on the already-confirmed token must not "
+                + "have released the claim held by that roll");
+        assertEquals(LeadRollover.RefusalReason.ROLL_ALREADY_RUNNING, anotherDecision.reason());
+    }
+
+    @Test
+    @DisplayName("[SINGLE-FLIGHT 6] a continuationRunner that rejects the hand-off still releases "
+            + "the claim and leaves a terminal FAILED outcome, not a stuck IN_PROGRESS")
+    void continuationRunnerThatRejectsTheHandOffStillReleasesTheClaim() throws IOException {
+        FakeHerdr herdr = new FakeHerdr();
+        Path handover = writeHandover("handover contents");
+        AtomicLong clock = new AtomicLong(1_000);
+        AgentControl agents = new AgentControl(herdr);
+        // A continuationRunner that rejects every hand-off, standing in for e.g. a bounded
+        // executor's RejectedExecutionException — the continuation never runs, so runRollover's
+        // own finally never gets a chance to release the claim either.
+        LeadRollover rollover = new LeadRollover(agents, () -> cfg(handover.toString()), _ -> null,
+                fixedClock(clock), () -> { },
+                r -> { throw new RuntimeException("simulated continuationRunner rejection"); });
+
+        LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full");
+        LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true);
+
+        assertTrue(decision.accepted(), "every gate passed before the hand-off itself rejected — "
+                + "confirm() treats that rejection the same way it treats a throw INSIDE the "
+                + "continuation: logged and surfaced only through status(), never as a refusal here: "
+                + decision.reason() + " / " + decision.detail());
+
+        LeadRollover.RollStatus status = rollover.status(pending.token());
+        assertEquals(LeadRollover.RollState.FAILED, status.state(), "a rejected hand-off must leave "
+                + "a TERMINAL outcome, never a stuck IN_PROGRESS — status() would otherwise have no "
+                + "way to tell a roll that never started from one still genuinely running: got "
+                + status.state() + " / " + status.detail());
+        assertNotEquals(LeadRollover.RollState.IN_PROGRESS, status.state());
+
+        LeadRollover.PendingRollover retry = rollover.open(LEAD, "retry after the rejected hand-off");
+        LeadRollover.RollDecision retryDecision = rollover.confirm(LEAD, retry.token(), true);
+        assertTrue(retryDecision.accepted(), "the claim must have been released even though "
+                + "continuationRunner itself threw before the continuation ever ran — otherwise "
+                + "this lead terminal could never be rolled again: " + retryDecision.reason() + " / "
+                + retryDecision.detail());
+    }
 }

fleetd/src/test/java/dev/ltms/fleet/msg/MessageServiceTest.java

@@ -888,6 +888,39 @@ class MessageServiceTest {
         assertNull(messages.poll("task-999999"), "a ticket that was never minted is unknown");
     }
 
+    // --- fleetd #719: a per-boot nonce keeps one instance's ticket ids out of another's space ---
+
+    /** A second, fully independent instance — its own agents/injector/rendezvous/inbox, not shared. */
+    private MessageService newIndependentInstance() {
+        FakeHerdr otherHerdr = new FakeHerdr().readText("BUILD GREEN: 391 files");
+        AgentControl otherAgents = new AgentControl(otherHerdr);
+        Injector otherInjector = new Injector(otherAgents);
+        return new MessageService(otherAgents, otherInjector, new Rendezvous(), new InMemoryReplyInbox());
+    }
+
+    @Test
+    void twoInstancesMintDisjointTicketIds() {
+        MessageService other = newIndependentInstance();
+        String ticketFromThis = messages.sendAsync(T, "task on first instance", null, null);
+        String ticketFromOther = other.sendAsync(T, "task on second instance", null, null);
+        assertNotEquals(ticketFromThis, ticketFromOther,
+                "each instance mints its own id space, so even a first ticket from each must differ");
+    }
+
+    @Test
+    void foreignInstanceTicketDoesNotResolve() {
+        MessageService other = newIndependentInstance();
+        String ticket = messages.sendAsync(T, "task on first instance", null, null);
+        // `other` must reach the same sequence number, or this test passes against an empty map
+        // instead of against a colliding id.
+        other.sendAsync(T, "task on second instance", null, null);
+
+        // control: the id resolves in the instance that minted it, so a null below cannot be
+        // explained by broken plumbing — only by the ticket being foreign to `other`.
+        assertNotNull(messages.poll(ticket), "the minting instance must still resolve its own ticket");
+        assertNull(other.poll(ticket), "a ticket minted by a different instance must not resolve here");
+    }
+
     // --- fleetd #705: a ticket's creator terminal gates who may poll it -------------------------
 
     @Test