From b874afb0af5d9966c35f82b4ab4076196fc60add Mon Sep 17 00:00:00 2001 From: Dai Ha Date: Sat, 19 Sep 2026 16:17:39 +0700 Subject: [PATCH 1/2] fleetd: make lead-rollover outcomes readable after confirm() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LeadRollover previously logged every post-confirm() failure only — a lead has no way to read the daemon log, so a roll that timed out because its own turn never settled (or /clear never re-settled) was invisible; the lead would carry on believing a fresh session was coming. Add a bounded (cap=200) token -> outcome record, written at each of the three exits in runRollover (ROLLED, TURN_NEVER_SETTLED, CLEAR_NEVER_SETTLED), and a read-only LeadRollover#status(token) accessor. The TURN_NEVER_SETTLED detail names turnSettleSeconds explicitly so a reader knows what to raise. Wire a "status" action onto the fleet_handover MCP tool (handler + schema); it never schedules, cancels, or retries anything — confirm() remains the only path that can ever cause a /clear. Extends LeadRolloverTest (33 -> 39 tests) covering the six acceptance properties, and FleetMcpHandoverTest (8 -> 12) for the new tool action. --- .../dev/ltms/fleet/lead/LeadRollover.java | 127 +++++++++++++ .../java/dev/ltms/fleet/mcp/FleetMcp.java | 52 ++++-- .../dev/ltms/fleet/lead/LeadRolloverTest.java | 171 ++++++++++++++++++ .../ltms/fleet/mcp/FleetMcpHandoverTest.java | 46 +++++ 4 files changed, 385 insertions(+), 11 deletions(-) diff --git a/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java b/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java index c7039af..b5e3d3f 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java +++ b/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java @@ -10,6 +10,8 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -181,6 +183,68 @@ public final class LeadRollover { } } + /** + * How many finished tokens {@link #outcomes} remembers before it starts evicting the oldest — + * bounded so a long-running daemon never grows this map without limit. Chosen generously + * rather than tightly: production rolls are rare (this class's own ticket found exactly ONE + * completed roll ever logged on this host), and each entry is a handful of short strings, so + * even a full cap costs a few tens of kilobytes — nowhere near a reason to make it configurable. + * 200 entries comfortably outlasts any operator's own memory of "did that roll I asked for + * actually happen", which is the whole reason {@link #status} exists. + */ + static final int OUTCOME_HISTORY_CAP = 200; + + /** + * What is known about one token, right now — the answer {@link #status} gives. Distinguishes + * three terminal outcomes an approved roll can finish with, plus the two non-terminal answers: + * still pending, or nothing known about this token at all. + */ + public enum RollState { + /** + * {@code token} is still open: either {@link #open} was called and {@link #confirm} has not + * been (or not successfully) yet, or a {@link #confirm} call failed one of its gate checks + * and left the token pending for a retry — see {@link #confirm}'s javadoc ("token stays + * pending"). Indistinguishable from a genuinely fresh request; a caller wanting to know + * WHICH gate most recently refused should read the {@link RollDecision} that {@link + * #confirm} itself returned, not this status. + */ + PENDING, + /** + * {@link #confirm} was approved and the deferred continuation completed the entire roll: + * the calling lead's turn settled, {@code /clear} was sent and settled, and {@code + * bootstrapText} was sent. + */ + ROLLED, + /** + * {@link #confirm} was approved, but the calling lead's own turn never reached a boundary + * (IDLE or DONE) within {@code turnSettleSeconds} — no {@code /clear} was ever sent, at + * all. This is the branch the fleetd #480 correction exists to make safe, and the one this + * status exists to make VISIBLE: before this, a lead that hit this case had no way to find + * out, and would carry on believing it was about to be replaced. See this class's javadoc. + */ + TURN_NEVER_SETTLED, + /** + * {@link #confirm} was approved and {@code /clear} was sent, but the pane never re-settled + * within {@code clearSettleSeconds} — {@code bootstrapText} was never sent. + */ + CLEAR_NEVER_SETTLED, + /** + * {@code token} names nothing this instance currently knows about: never issued by {@link + * #open}, dropped by {@link #cancel}, or aged out of {@link #outcomes}'s bounded history. + * These three causes are not distinguished — all of them mean "there is nothing to tell + * you", which is the entire content of a clean answer here. + */ + UNKNOWN + } + + /** + * The answer {@link #status} gives for one token: a {@link RollState} and a human-readable + * {@code detail}. For {@link RollState#TURN_NEVER_SETTLED}, {@code detail} names {@code + * turnSettleSeconds} and its configured value explicitly, so a reader who sees this knows what + * to raise. + */ + public record RollStatus(RollState state, String detail) {} + private final AgentControl agents; private final Supplier configSupplier; /** @@ -201,6 +265,23 @@ public final class LeadRollover { */ private final Consumer continuationRunner; private final Map pending = new ConcurrentHashMap<>(); + /** + * Finished tokens → what actually happened, for {@link #status}. Bounded by {@link + * #OUTCOME_HISTORY_CAP}, oldest evicted first ({@code removeEldestEntry} on an insertion-order + * {@link LinkedHashMap}). Wrapped in {@link Collections#synchronizedMap} because entries are + * written from whatever thread {@code continuationRunner} runs the roll on (a fresh virtual + * thread in production, the calling test thread under {@code Runnable::run}) and read from + * whatever thread calls {@link #status} (the MCP handler thread) — a plain {@code + * LinkedHashMap} is not safe for that, and {@code removeEldestEntry} additionally requires + * external synchronization even for a thread-safe map that merely wraps it. + */ + private final Map outcomes = Collections.synchronizedMap( + new LinkedHashMap<>(16, 0.75f, false) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > OUTCOME_HISTORY_CAP; + } + }); /** Production constructor — wall clock, real sleep between settle polls, a real virtual thread. */ public LeadRollover(AgentControl agents, Supplier configSupplier, @@ -393,6 +474,11 @@ public final class LeadRollover { + "turn is still live and clearing it now would destroy live context " + "(token={}, configured={}s elapsed={}ms)", lead, p.token(), cfg.turnSettleSeconds(), turnResult.elapsedMillis()); + outcomes.put(p.token(), new RollStatus(RollState.TURN_NEVER_SETTLED, + "the calling lead's own turn never reached a boundary (IDLE or DONE) within " + + "turnSettleSeconds=" + cfg.turnSettleSeconds() + "s (measured elapsed=" + + turnResult.elapsedMillis() + "ms) — no /clear was ever sent. If this " + + "keeps happening, raise turnSettleSeconds in fleetd.yaml")); return; } @@ -411,11 +497,17 @@ public final class LeadRollover { + "elapsed={}ms nudges={})", lead, p.token(), cfg.clearSettleSeconds(), clearResult.elapsedMillis(), clearResult.nudges()); + outcomes.put(p.token(), new RollStatus(RollState.CLEAR_NEVER_SETTLED, + "/clear was sent, but the pane never re-settled within clearSettleSeconds=" + + cfg.clearSettleSeconds() + "s (measured elapsed=" + clearResult.elapsedMillis() + + "ms, nudges=" + clearResult.nudges() + ") — bootstrapText was never sent")); return; } agents.send(lead, cfg.bootstrapTextFor(p.handoverPath())); long rollElapsedMillis = nowMillis.getAsLong() - rollStartMillis; log.info("lead-rollover: rolled token={} lead={} elapsedMs={}", p.token(), lead, rollElapsedMillis); + outcomes.put(p.token(), new RollStatus(RollState.ROLLED, + "rolled successfully in " + rollElapsedMillis + "ms")); } /** Drop a pending request without rolling. @return whether a pending request existed for {@code token} */ @@ -423,6 +515,41 @@ public final class LeadRollover { return pending.remove(token) != null; } + /** + * Read-only: what is currently known about {@code token}. Never sends anything, never + * schedules, cancels, or retries a roll — a caller may poll this as often as it likes + * with no side effect at all, which is exactly why it exists: every failure past {@link + * #confirm} used to be a {@code log.warn} a lead can never read (see this class's javadoc), and + * this is the only route back. + * + * @param token the token {@link #open} returned; {@code null} or blank is a clean {@link + * RollState#UNKNOWN}, never a {@link NullPointerException} — {@link #pending} is a + * {@link ConcurrentHashMap}, which throws on a {@code null} key lookup, so this + * short-circuits before ever reaching it + * @return {@link RollState#PENDING} while {@code token} is still open (including one left + * pending by a {@link #confirm} gate refusal — see that method's javadoc); a terminal + * state once the approved roll's continuation has finished, read from the bounded + * {@link #outcomes} history; or {@link RollState#UNKNOWN} for a token never issued, + * cancelled, or aged out of that history + */ + public RollStatus status(String token) { + if (token == null || token.isBlank()) { + return new RollStatus(RollState.UNKNOWN, "no token given"); + } + if (pending.containsKey(token)) { + return new RollStatus(RollState.PENDING, "open() has been called for this token and " + + "it has not yet been confirmed — or a confirm() gate check failed and left it " + + "pending, so the same token may be retried once the problem is fixed"); + } + RollStatus finished = outcomes.get(token); + if (finished != null) { + return finished; + } + return new RollStatus(RollState.UNKNOWN, "token names no pending or finished rollover " + + "request known to this instance — never issued, cancelled, or aged out of the " + + "bounded history (cap=" + OUTCOME_HISTORY_CAP + ")"); + } + /** * The three handover-file checks, in order: exists, not empty, fresh (modified after * {@link #open}'s timestamp and not older than {@code maxDocAgeSeconds}). Stats {@code diff --git a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java index 6fa0559..066d203 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java +++ b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java @@ -1248,14 +1248,16 @@ public final class FleetMcp { Map args) { String action = str(args, "action"); if (isBlank(action)) { - return error("action is required: \"open\", \"confirm\" or \"cancel\""); + return error("action is required: \"open\", \"confirm\", \"cancel\" or \"status\""); } return switch (action) { case "open" -> handoverOpen(leadRollover, callerTerminal, str(args, "reason")); case "confirm" -> handoverConfirm(leadRollover, callerTerminal, str(args, "token"), truthy(args, "operatorConfirmed")); case "cancel" -> handoverCancel(leadRollover, str(args, "token")); - default -> error("unknown action \"" + action + "\" — must be \"open\", \"confirm\" or \"cancel\""); + case "status" -> handoverStatus(leadRollover, str(args, "token")); + default -> error("unknown action \"" + action + + "\" — must be \"open\", \"confirm\", \"cancel\" or \"status\""); }; } @@ -1325,6 +1327,29 @@ public final class FleetMcp { return text(json(m)); } + /** + * {@code action: "status"}. Read-only — see {@link LeadRollover#status}: never schedules, + * cancels, or retries anything, and it is the only way for a lead to find out what happened to + * a token past {@code confirm()}, since every outcome after that point is otherwise logged only + * (see {@link LeadRollover}'s class javadoc). + */ + private static McpSchema.CallToolResult handoverStatus(LeadRollover leadRollover, String token) { + if (leadRollover == null) { + Map m = new LinkedHashMap<>(); + m.put("state", "NOT_CONFIGURED"); + m.put("detail", "leadRollover: is not configured"); + return text(json(m)); + } + if (isBlank(token)) { + return error("token is required for action \"status\""); + } + LeadRollover.RollStatus s = leadRollover.status(token); + Map m = new LinkedHashMap<>(); + m.put("state", s.state().name()); + m.put("detail", s.detail()); + return text(json(m)); + } + /** The one shared {@code NOT_CONFIGURED} refusal shape for {@code open}/{@code confirm}. */ private static McpSchema.CallToolResult notConfigured() { return refusalJson(false, "NOT_CONFIGURED", "leadRollover: is not configured"); @@ -2265,20 +2290,25 @@ public final class FleetMcp { return tool(FleetTool.HANDOVER.wireName(), "Replace your OWN lead session once its context is full: write a handover file, " + "then use this to have fleetd clear your pane and bootstrap a fresh lead " - + "session against it. Three actions: 'open' (requests a token and the " + + "session against it. Four actions: 'open' (requests a token and the " + "handoverPath you must write the handover file to before confirming), " + "'confirm' (validates every gate and — only if every one passes — schedules " + "the roll; it does NOT itself clear the pane, the roll runs once this call's " - + "own turn ends), and 'cancel' (drops a pending request without rolling). " - + "Primary-only. There is deliberately no terminal/session/leadTerminal " - + "parameter: the pane to roll is always resolved from YOUR OWN connection, " - + "never a value you pass, so you can only ever roll yourself — never another " - + "lead. Requires leadRollover: to be configured; when it is not, every action " - + "returns a clean refusal naming NOT_CONFIGURED instead of failing.", + + "own turn ends), 'cancel' (drops a pending request without rolling), and " + + "'status' (read-only: what happened to a token after 'confirm' — the roll " + + "completed, the calling turn never settled within turnSettleSeconds so no " + + "/clear was ever sent, or /clear itself never settled so bootstrapText was " + + "never sent; never schedules, cancels or retries anything). Primary-only. " + + "There is deliberately no terminal/session/leadTerminal parameter: the pane " + + "to roll is always resolved from YOUR OWN connection, never a value you " + + "pass, so you can only ever roll yourself — never another lead. Requires " + + "leadRollover: to be configured; when it is not, every action returns a " + + "clean refusal naming NOT_CONFIGURED instead of failing.", objectSchema(Map.of( - "action", stringProp("\"open\", \"confirm\" or \"cancel\""), + "action", stringProp("\"open\", \"confirm\", \"cancel\" or \"status\""), "reason", stringProp("Free-text audit note for \"open\" (optional, logged only)"), - "token", stringProp("The token \"open\" returned — required for \"confirm\" and \"cancel\""), + "token", stringProp("The token \"open\" returned — required for \"confirm\", " + + "\"cancel\" and \"status\""), "operatorConfirmed", Map.of("type", "boolean", "description", "For \"confirm\": your answer to \"has the human operator " + "confirmed this wipe\" (default false; only consulted when " diff --git a/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java b/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java index 5eabaf7..da39888 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java @@ -1004,6 +1004,177 @@ class LeadRolloverTest { } } + // ---- CB-... : LeadRollover#status makes the outcome of a confirmed roll readable ----------- + + @Test + @DisplayName("[STATUS 1] after the calling turn never settles, status() reports " + + "TURN_NEVER_SETTLED for that token — this assertion could not even be written before " + + "status() existed") + void statusReportsTurnNeverSettledAfterTheRollIsAbandoned() throws IOException { + FakeHerdr herdr = new FakeHerdr(); + herdr.agentStatus("working"); // the calling lead's own pane — never goes idle in this test + Path handover = writeHandover("handover contents"); + FleetConfig.LeadRollover config = + new FleetConfig.LeadRollover(handover.toString(), true, 3600, 1 /*turnSettleSeconds*/, 20, "text"); + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(herdr, config, () -> clock.addAndGet(500)); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "every synchronous gate should pass; the refusal happens " + + "only inside the deferred continuation, which this test's synchronous runner has " + + "already run to completion by the time confirm() returns"); + assertEquals(0, promptCallCount(herdr), "sanity: /clear was never sent"); + + LeadRollover.RollStatus status = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.TURN_NEVER_SETTLED, status.state()); + assertTrue(status.detail().contains("turnSettleSeconds"), "the detail must name the knob a " + + "reader needs to raise: " + status.detail()); + assertTrue(status.detail().toLowerCase().contains("no /clear"), "the detail must say plainly " + + "that no /clear was ever sent: " + status.detail()); + } + + @Test + @DisplayName("[STATUS 2] a roll that completes reports ROLLED for its token") + void statusReportsRolledForACompletedRoll() throws IOException { + FakeHerdr herdr = new FakeHerdr(); // default idle — a full successful roll + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), fixedClock(clock)); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail()); + assertEquals(2, promptCallCount(herdr), "sanity: /clear then bootstrapText were both sent"); + + LeadRollover.RollStatus status = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.ROLLED, status.state()); + assertNotNull(status.detail()); + } + + @Test + @DisplayName("[STATUS 3] a roll where /clear never settles reports CLEAR_NEVER_SETTLED — " + + "distinct from both ROLLED and TURN_NEVER_SETTLED") + void statusReportsClearNeverSettledDistinctFromTheOtherTwoStates() throws IOException { + // Idle until /clear is sent, then permanently working — the SECOND wait never settles. + FakeHerdr fake = new FakeHerdr(); + HerdrClient flipsAfterClear = new HerdrClient() { + @Override + public JsonNode call(String method, Object params) throws HerdrException { + JsonNode result = fake.call(method, params); + if ("agent.prompt".equals(method) && String.valueOf(params).contains("/clear")) { + fake.agentStatus("working"); + } + return result; + } + + @Override + public void close() { + fake.close(); + } + }; + Path handover = writeHandover("handover contents"); + FleetConfig.LeadRollover config = + new FleetConfig.LeadRollover(handover.toString(), true, 3600, 20, 1 /*clearSettleSeconds*/, "boot text"); + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(flipsAfterClear, config, () -> clock.addAndGet(500)); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "every synchronous gate passes; the refusal is logged only, " + + "deep inside the deferred continuation"); + assertEquals(1, promptCallCount(fake), "sanity: only /clear was sent, never bootstrapText"); + + LeadRollover.RollStatus status = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.CLEAR_NEVER_SETTLED, status.state()); + assertNotEquals(LeadRollover.RollState.ROLLED, status.state()); + assertNotEquals(LeadRollover.RollState.TURN_NEVER_SETTLED, status.state()); + assertTrue(status.detail().contains("clearSettleSeconds"), status.detail()); + } + + @Test + @DisplayName("[STATUS 4] a token that was never issued, or was cancelled, gives a clean " + + "UNKNOWN answer rather than an exception or a false ROLLED") + void statusOnUnissuedOrCancelledTokenIsCleanNotAnExceptionOrFalseSuccess() throws IOException { + FakeHerdr herdr = new FakeHerdr(); + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), fixedClock(clock)); + + // never issued at all + LeadRollover.RollStatus neverIssued = assertDoesNotThrow(() -> rollover.status("no-such-token")); + assertEquals(LeadRollover.RollState.UNKNOWN, neverIssued.state()); + assertNotEquals(LeadRollover.RollState.ROLLED, neverIssued.state()); + + // null / blank must not throw either — pending is a ConcurrentHashMap, which throws on a + // null-key lookup unless status() guards it first + assertEquals(LeadRollover.RollState.UNKNOWN, assertDoesNotThrow(() -> rollover.status(null)).state()); + assertEquals(LeadRollover.RollState.UNKNOWN, assertDoesNotThrow(() -> rollover.status(" ")).state()); + + // opened, then cancelled + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + assertTrue(rollover.cancel(pending.token())); + LeadRollover.RollStatus cancelled = assertDoesNotThrow(() -> rollover.status(pending.token())); + assertEquals(LeadRollover.RollState.UNKNOWN, cancelled.state()); + assertNotEquals(LeadRollover.RollState.ROLLED, cancelled.state()); + } + + @Test + @DisplayName("[STATUS 5] the bounded outcome history never grows past its cap") + void statusHistoryDoesNotGrowPastItsCap() throws IOException { + FakeHerdr herdr = new FakeHerdr(); // default idle — every roll completes + Path handover = writeHandover("handover contents"); // one file, reused by every roll below — + // checkHandover only compares its mtime against each open()'s OWN requestedAtMillis (the + // fake clock, in the low thousands), and the file's real (wall-clock) mtime is always far + // larger than that, so freshness passes on every iteration without rewriting the file. + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), () -> clock.addAndGet(1)); + + int rolls = LeadRollover.OUTCOME_HISTORY_CAP + 50; + String[] tokens = new String[rolls]; + for (int i = 0; i < rolls; i++) { + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full #" + i); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "roll #" + i + " should have been approved: " + decision.detail()); + tokens[i] = pending.token(); + } + + assertEquals(LeadRollover.RollState.ROLLED, rollover.status(tokens[rolls - 1]).state(), + "the most recently finished roll's outcome must still be in the bounded history"); + + // There is no direct size accessor for the bounded history, so boundedness is asserted + // indirectly and behaviourally: the OLDEST finished roll's outcome must have been evicted + // (reads back as a clean UNKNOWN, exactly like a token that was never issued) once more than + // OUTCOME_HISTORY_CAP rolls have gone through this instance. If the cap were not enforced, + // tokens[0] would still read back ROLLED here, and this assertion would fail. + assertEquals(LeadRollover.RollState.UNKNOWN, rollover.status(tokens[0]).state(), + "the oldest finished roll's outcome must have been evicted once the cap was " + + "exceeded — otherwise the bounded history is not actually bounded"); + } + + @Test + @DisplayName("[STATUS 6] a status() call against a pending roll is read-only — no herdr call is " + + "made and the pending request is left untouched") + void statusCallAgainstAPendingRollIsReadOnlyAndTouchesNothing() throws IOException { + FakeHerdr herdr = new FakeHerdr(); + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + LeadRollover rollover = newRollover(herdr, cfg(handover.toString()), fixedClock(clock)); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + + LeadRollover.RollStatus status = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.PENDING, status.state()); + assertEquals(0, herdr.calls.size(), "status() must never make any herdr call at all — not " + + "just no agent.prompt — since it must never schedule, cancel, or retry anything"); + + // the pending request must be left exactly as it was: the SAME token can still be confirmed + // afterwards, as if status() had never been called. + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "status() must not have consumed or otherwise disturbed the " + + "pending request: " + decision.reason() + " / " + decision.detail()); + } + @Test @DisplayName("[fleetd #494 follow-up] the turn-settle timeout warn line prints the MEASURED " + "elapsed time next to the configured budget, never the configured value alone") diff --git a/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetMcpHandoverTest.java b/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetMcpHandoverTest.java index a93a129..eebad38 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetMcpHandoverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetMcpHandoverTest.java @@ -258,5 +258,51 @@ class FleetMcpHandoverTest { assertTrue(textOf(r).contains("\"cancelled\":true"), textOf(r)); } + // --- new: action "status" — makes the outcome of a confirm() readable through the tool ---- + + @Test + @DisplayName("status on a null LeadRollover is a clean NOT_CONFIGURED refusal, never a throw") + void statusWithNullLeadRolloverRefusesCleanly() { + McpSchema.CallToolResult r = assertDoesNotThrow(() -> FleetMcp.handover(null, LEAD, + Map.of("action", "status", "token", "whatever"))); + assertFalse(r.isError()); + assertTrue(textOf(r).contains("NOT_CONFIGURED"), textOf(r)); + } + + @Test + @DisplayName("status on a token that was never opened reports UNKNOWN") + void statusOnUnknownTokenReportsUnknown() { + LeadRollover rollover = newRollover(tmp.resolve("h.md").toString()); + + McpSchema.CallToolResult r = FleetMcp.handover(rollover, LEAD, + Map.of("action", "status", "token", "does-not-exist")); + assertFalse(r.isError()); + assertTrue(textOf(r).contains("\"state\":\"UNKNOWN\""), textOf(r)); + } + + @Test + @DisplayName("status on a token that is still pending (opened, not confirmed) reports PENDING") + void statusOnPendingTokenReportsPending() { + LeadRollover rollover = newRollover(tmp.resolve("h.md").toString()); + String token = extractToken(textOf(FleetMcp.handover(rollover, LEAD, Map.of("action", "open")))); + + McpSchema.CallToolResult r = FleetMcp.handover(rollover, LEAD, + Map.of("action", "status", "token", token)); + assertFalse(r.isError()); + assertTrue(textOf(r).contains("\"state\":\"PENDING\""), textOf(r)); + } + + @Test + @DisplayName("status is registered on the tool's schema and the schema still names no caller-identity parameter") + void statusActionIsAdvertisedOnTheSchema() { + FleetMcp m = mcp(null); + McpSchema.Tool tool = m.registeredTools().stream() + .filter(t -> "fleet_handover".equals(t.name())) + .findFirst() + .orElseThrow(() -> new AssertionError("fleet_handover was not registered")); + assertTrue(tool.description().contains("'status'"), + "the tool's own description must advertise the 'status' action: " + tool.description()); + } + // --- acceptance 7 (wiring) is covered by FleetdLeadRolloverWiringTest, unchanged ----------- } From bf895616a5b2a9997c8bb82621cf98b0959e9367 Mon Sep 17 00:00:00 2001 From: Dai Ha Date: Sat, 19 Sep 2026 22:31:47 +0700 Subject: [PATCH 2/2] fleetd: distinguish an in-flight roll from an unknown token confirm() removed the token from pending before handing the roll to the continuation, and outcomes was only written when runRollover reached an exit. For the whole duration of the roll the token was in neither map, so status() answered UNKNOWN - documented as "never issued, cancelled, or aged out". A lead polling right after its own confirm was told the roll had never been requested. Adds RollState.IN_PROGRESS, written at the confirm hand-off rather than at the roll's end, so there is no gap. status() now reads outcomes before pending, so the hand-off write cannot race the removal. Also corrects the class javadoc, which still claimed nothing calls this class. Recovered by the lead: the authoring member ended on a backend error (the host slept mid-response) with this work uncommitted in its worktree. Verified before committing: mvn -o clean install exit 0, 1819 tests, 0 failures, 0 errors, 0 skipped, 143 reports; LeadRolloverTest 43 (was 39). --- .../dev/ltms/fleet/lead/LeadRollover.java | 84 +++++++--- .../java/dev/ltms/fleet/mcp/FleetMcp.java | 9 +- .../dev/ltms/fleet/lead/LeadRolloverTest.java | 148 ++++++++++++++++++ 3 files changed, 215 insertions(+), 26 deletions(-) diff --git a/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java b/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java index b5e3d3f..aefa43f 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java +++ b/fleetd/src/main/java/dev/ltms/fleet/lead/LeadRollover.java @@ -27,9 +27,11 @@ import java.util.function.Supplier; * every gate ({@link #confirm}'s own checks) has passed — a deferred, single-shot continuation * clears the lead's own pane and bootstraps a fresh session against that file. * - *

This is the executor only. Nothing in this ticket wires an MCP tool onto {@link #open}/ - * {@link #confirm}/{@link #cancel} — that is a separate, later unit; until it lands, nothing calls - * this class at all. + *

This is the executor behind the {@code fleet_handover} MCP tool ({@code + * dev.ltms.fleet.mcp.FleetMcp#handover}), which drives {@link #open}, {@link #confirm}, {@link + * #cancel}, and {@link #status} from a tool call — wired in fleetd #480 Unit C. An earlier + * version of this paragraph said nothing called this class at all; that stopped being true once + * that unit landed, and this correction exists so the javadoc does not go on claiming it. * *

{@code confirm()} cannot roll inline — a fleetd #480 correction. The first * version of this class called {@code agents.send(lead, "/clear")} directly from inside {@code @@ -184,20 +186,31 @@ public final class LeadRollover { } /** - * How many finished tokens {@link #outcomes} remembers before it starts evicting the oldest — - * bounded so a long-running daemon never grows this map without limit. Chosen generously - * rather than tightly: production rolls are rare (this class's own ticket found exactly ONE - * completed roll ever logged on this host), and each entry is a handful of short strings, so - * even a full cap costs a few tens of kilobytes — nowhere near a reason to make it configurable. - * 200 entries comfortably outlasts any operator's own memory of "did that roll I asked for - * actually happen", which is the whole reason {@link #status} exists. + * How many tokens {@link #outcomes} remembers before it starts evicting the oldest — bounded + * so a long-running daemon never grows this map without limit. Chosen generously rather than + * tightly: production rolls are rare (this class's own ticket found exactly ONE completed roll + * ever logged on this host), and each entry is a handful of short strings, so even a full cap + * costs a few tens of kilobytes — nowhere near a reason to make it configurable. 200 entries + * comfortably outlasts any operator's own memory of "did that roll I asked for actually + * happen", which is the whole reason {@link #status} exists. + * + *

This cap counts {@link RollState#IN_PROGRESS} entries exactly the same as + * finished ones. There is only the one bounded map: {@link #confirm} writes an {@link + * RollState#IN_PROGRESS} entry into {@link #outcomes} at hand-off, and the deferred + * continuation later overwrites that SAME key with a terminal state — it never inserts a + * second entry. An approved roll therefore occupies one slot in this map for its entire + * lifetime, from the moment {@link #confirm} hands off, not only once it finishes; a + * confirmed-but-not-yet-finished roll counts against the cap exactly like a finished one. The + * alternative (a separate, uncapped in-flight map) would let a burst of confirmed-but-stuck + * rolls grow without bound — the exact failure this cap exists to prevent — so it was rejected. */ static final int OUTCOME_HISTORY_CAP = 200; /** * What is known about one token, right now — the answer {@link #status} gives. Distinguishes - * three terminal outcomes an approved roll can finish with, plus the two non-terminal answers: - * still pending, or nothing known about this token at all. + * three terminal outcomes an approved roll can finish with, one in-flight outcome for a roll + * that has been approved but has not finished yet, and two answers for a token that names no + * active work at all: still pending confirmation, or nothing known about this token at all. */ public enum RollState { /** @@ -206,9 +219,21 @@ public final class LeadRollover { * and left the token pending for a retry — see {@link #confirm}'s javadoc ("token stays * pending"). Indistinguishable from a genuinely fresh request; a caller wanting to know * WHICH gate most recently refused should read the {@link RollDecision} that {@link - * #confirm} itself returned, not this status. + * #confirm} itself returned, not this status. Never the state of an APPROVED + * roll — see {@link #IN_PROGRESS}, which {@link #confirm} records at the moment it + * hands off, before this token is even removed from the pending set. */ PENDING, + /** + * {@link #confirm} approved this roll and handed it to the deferred continuation, which has + * not finished yet. Recorded by {@link #confirm} itself, at hand-off — before + * {@code token} is removed from the pending set — so there is never a gap in which {@link + * #status} could wrongly answer {@link #UNKNOWN} ("nothing was ever requested") for a roll + * that is, in fact, actively running. This is not sticky: the deferred continuation + * overwrites this same entry with a terminal state ({@link #ROLLED}, {@link + * #TURN_NEVER_SETTLED}, or {@link #CLEAR_NEVER_SETTLED}) once it finishes. + */ + IN_PROGRESS, /** * {@link #confirm} was approved and the deferred continuation completed the entire roll: * the calling lead's turn settled, {@code /clear} was sent and settled, and {@code @@ -447,6 +472,15 @@ public final class LeadRollover { return docCheck; } + // Record IN_PROGRESS BEFORE removing from `pending` — see RollState#IN_PROGRESS and + // OUTCOME_HISTORY_CAP's javadoc. This ordering means `token` is written into `outcomes` + // while it is STILL present in `pending`; status() checks `outcomes` first (see that + // method), so it reports IN_PROGRESS immediately, not the brief-but-real gap a + // remove-then-put ordering would leave in which the token is in neither map. + outcomes.put(token, new RollStatus(RollState.IN_PROGRESS, + "confirm() approved this roll and handed it to the deferred continuation; it has " + + "not finished yet — still waiting for the calling turn to settle, for " + + "/clear to be sent and settle, or for bootstrapText to be sent")); pending.remove(token); log.info("lead-rollover: confirmed token={} lead={} — roll scheduled once the calling turn ends", token, callerTerminal); @@ -526,25 +560,31 @@ public final class LeadRollover { * RollState#UNKNOWN}, never a {@link NullPointerException} — {@link #pending} is a * {@link ConcurrentHashMap}, which throws on a {@code null} key lookup, so this * short-circuits before ever reaching it - * @return {@link RollState#PENDING} while {@code token} is still open (including one left - * pending by a {@link #confirm} gate refusal — see that method's javadoc); a terminal - * state once the approved roll's continuation has finished, read from the bounded - * {@link #outcomes} history; or {@link RollState#UNKNOWN} for a token never issued, - * cancelled, or aged out of that history + * @return {@link RollState#IN_PROGRESS} for an approved roll whose continuation has not + * finished yet, or a terminal state once it has (both read from {@link #outcomes} — + * checked FIRST, see below); {@link RollState#PENDING} while {@code token} is still + * open and has not yet been approved (including one left pending by a {@link #confirm} + * gate refusal — see that method's javadoc); or {@link RollState#UNKNOWN} for a token + * never issued, cancelled, or aged out of the bounded history */ public RollStatus status(String token) { if (token == null || token.isBlank()) { return new RollStatus(RollState.UNKNOWN, "no token given"); } + // `outcomes` is checked BEFORE `pending`, deliberately: `confirm` writes an IN_PROGRESS + // entry into `outcomes` before it removes `token` from `pending` (see `confirm`'s own + // comment at that call site), so for the brief window where a token is present in BOTH + // maps, this order reports the more accurate answer (IN_PROGRESS, already approved) rather + // than the stale one (PENDING, not yet approved) a pending-first check would give. + RollStatus recorded = outcomes.get(token); + if (recorded != null) { + return recorded; + } if (pending.containsKey(token)) { return new RollStatus(RollState.PENDING, "open() has been called for this token and " + "it has not yet been confirmed — or a confirm() gate check failed and left it " + "pending, so the same token may be retried once the problem is fixed"); } - RollStatus finished = outcomes.get(token); - if (finished != null) { - return finished; - } return new RollStatus(RollState.UNKNOWN, "token names no pending or finished rollover " + "request known to this instance — never issued, cancelled, or aged out of the " + "bounded history (cap=" + OUTCOME_HISTORY_CAP + ")"); diff --git a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java index 066d203..34b8a54 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java +++ b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java @@ -2295,10 +2295,11 @@ public final class FleetMcp { + "'confirm' (validates every gate and — only if every one passes — schedules " + "the roll; it does NOT itself clear the pane, the roll runs once this call's " + "own turn ends), 'cancel' (drops a pending request without rolling), and " - + "'status' (read-only: what happened to a token after 'confirm' — the roll " - + "completed, the calling turn never settled within turnSettleSeconds so no " - + "/clear was ever sent, or /clear itself never settled so bootstrapText was " - + "never sent; never schedules, cancels or retries anything). Primary-only. " + + "'status' (read-only: what happened to a token after 'confirm' — still " + + "running (approved but not finished yet), the roll completed, the calling " + + "turn never settled within turnSettleSeconds so no /clear was ever sent, or " + + "/clear itself never settled so bootstrapText was never sent; never " + + "schedules, cancels or retries anything). Primary-only. " + "There is deliberately no terminal/session/leadTerminal parameter: the pane " + "to roll is always resolved from YOUR OWN connection, never a value you " + "pass, so you can only ever roll yourself — never another lead. Requires " diff --git a/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java b/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java index da39888..07d5bb8 100644 --- a/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java +++ b/fleetd/src/test/java/dev/ltms/fleet/lead/LeadRolloverTest.java @@ -16,6 +16,7 @@ import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -1175,6 +1176,153 @@ class LeadRolloverTest { + "pending request: " + decision.reason() + " / " + decision.detail()); } + // ---- PR #600 review round 2: IN_PROGRESS — the gap between confirm() handing off and the ---- + // ---- continuation finishing must never read back as UNKNOWN ("nothing was ever requested") -- + + /** + * A {@code continuationRunner} that CAPTURES the roll instead of running it, so a test can + * observe {@link LeadRollover#status} in the window between {@link LeadRollover#confirm} + * handing off and the roll actually finishing — the window the synchronous {@code + * Runnable::run} runner used everywhere else in this class collapses to nothing. Call {@link + * #runNext()} to finish exactly one held roll, once the test is done observing the in-flight + * state. + */ + private static final class HoldingRunner implements java.util.function.Consumer { + private final List held = new ArrayList<>(); + + @Override + public void accept(Runnable runnable) { + held.add(runnable); + } + + int heldCount() { + return held.size(); + } + + /** Runs (and removes) the oldest held roll — FIFO, matching confirm() call order. */ + void runNext() { + held.remove(0).run(); + } + } + + private static LeadRollover newRolloverWithHoldingRunner(HerdrClient herdr, + FleetConfig.LeadRollover config, LongSupplier nowMillis, HoldingRunner runner) { + AgentControl agents = new AgentControl(herdr); + return new LeadRollover(agents, () -> config, _ -> null, nowMillis, () -> { }, runner); + } + + @Test + @DisplayName("[IN-PROGRESS 1] between an approved confirm() and the continuation finishing, " + + "status() reports IN_PROGRESS — not UNKNOWN, and not PENDING") + void statusReportsInProgressBetweenConfirmAndTheContinuationFinishing() throws IOException { + FakeHerdr herdr = new FakeHerdr(); // default idle — the held roll WOULD complete once run + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + HoldingRunner runner = new HoldingRunner(); + LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()), + fixedClock(clock), runner); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail()); + assertEquals(1, runner.heldCount(), "sanity: the roll must have been handed to the " + + "continuation runner and held there, not run yet"); + assertEquals(0, promptCallCount(herdr), "sanity: the held continuation has not run, so " + + "nothing has been sent to the pane yet"); + + LeadRollover.RollStatus status = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.IN_PROGRESS, status.state(), + "a lead calling status() right after confirm() returned approved, while the roll " + + "is still running, must be told IN_PROGRESS — not UNKNOWN (\"nothing was " + + "ever requested\", which would wrongly invite it to call open() again " + + "mid-roll) and not PENDING (\"not yet approved\", which is simply false " + + "here): got " + status.state() + " / " + status.detail()); + } + + @Test + @DisplayName("[IN-PROGRESS 2] once the continuation finishes, the same token reports its " + + "terminal state — IN_PROGRESS is not sticky") + void inProgressStateIsNotStickyOnceTheContinuationFinishes() throws IOException { + FakeHerdr herdr = new FakeHerdr(); // default idle — the held roll completes once run + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + HoldingRunner runner = new HoldingRunner(); + LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()), + fixedClock(clock), runner); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail()); + assertEquals(LeadRollover.RollState.IN_PROGRESS, rollover.status(pending.token()).state(), + "sanity: must be IN_PROGRESS before the held continuation is run"); + + runner.runNext(); // finish the held roll now + + assertEquals(2, promptCallCount(herdr), "sanity: the roll actually ran to completion " + + "once released — /clear then bootstrapText"); + LeadRollover.RollStatus after = rollover.status(pending.token()); + assertEquals(LeadRollover.RollState.ROLLED, after.state(), + "the SAME token must now report its terminal state — IN_PROGRESS must not still be " + + "reported once the roll has actually finished"); + } + + @Test + @DisplayName("[IN-PROGRESS 4] IN_PROGRESS is distinct from every other RollState") + void inProgressStateIsDistinctFromAllOtherStates() throws IOException { + FakeHerdr herdr = new FakeHerdr(); + Path handover = writeHandover("handover contents"); + AtomicLong clock = new AtomicLong(1_000); + HoldingRunner runner = new HoldingRunner(); + LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()), + fixedClock(clock), runner); + + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full"); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "expected approval; got: " + decision.reason() + " / " + decision.detail()); + + LeadRollover.RollState inProgress = rollover.status(pending.token()).state(); + assertEquals(LeadRollover.RollState.IN_PROGRESS, inProgress); + for (LeadRollover.RollState other : LeadRollover.RollState.values()) { + if (other == LeadRollover.RollState.IN_PROGRESS) { + continue; + } + assertNotEquals(other, inProgress, "IN_PROGRESS must be distinct from " + other); + } + } + + @Test + @DisplayName("[IN-PROGRESS 5] eviction counts IN_PROGRESS entries toward the cap exactly like " + + "finished ones — a burst of confirmed-but-not-yet-finished rolls still ages out") + void evictionCountsInProgressEntriesTowardTheCap() throws IOException { + FakeHerdr herdr = new FakeHerdr(); + Path handover = writeHandover("handover contents"); // reused by every roll — see + // statusHistoryDoesNotGrowPastItsCap for why one shared file is enough for freshness. + AtomicLong clock = new AtomicLong(1_000); + HoldingRunner runner = new HoldingRunner(); // nothing run below — every roll stays IN_PROGRESS + LeadRollover rollover = newRolloverWithHoldingRunner(herdr, cfg(handover.toString()), + () -> clock.addAndGet(1), runner); + + int rolls = LeadRollover.OUTCOME_HISTORY_CAP + 50; + String[] tokens = new String[rolls]; + for (int i = 0; i < rolls; i++) { + LeadRollover.PendingRollover pending = rollover.open(LEAD, "context is full #" + i); + LeadRollover.RollDecision decision = rollover.confirm(LEAD, pending.token(), true); + assertTrue(decision.accepted(), "roll #" + i + " should have been approved: " + decision.detail()); + tokens[i] = pending.token(); + } + assertEquals(rolls, runner.heldCount(), "sanity: none of these rolls have been run — every " + + "one of them is sitting in outcomes as IN_PROGRESS, not a separate uncapped map"); + + assertEquals(LeadRollover.RollState.IN_PROGRESS, rollover.status(tokens[rolls - 1]).state(), + "the most recently confirmed (still in-flight) roll must still be in the bounded " + + "history"); + assertEquals(LeadRollover.RollState.UNKNOWN, rollover.status(tokens[0]).state(), + "the oldest confirmed roll's IN_PROGRESS entry must have been evicted once the cap " + + "was exceeded, exactly like a finished entry would be — proving IN_PROGRESS " + + "entries share the SAME bounded map and count against the SAME cap, rather " + + "than living in a second, uncapped in-flight map"); + } + @Test @DisplayName("[fleetd #494 follow-up] the turn-settle timeout warn line prints the MEASURED " + "elapsed time next to the configured budget, never the configured value alone")