diff --git a/fleetd/src/main/java/dev/ltms/fleet/Fleetd.java b/fleetd/src/main/java/dev/ltms/fleet/Fleetd.java index b7e0829..cc9191d 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/Fleetd.java +++ b/fleetd/src/main/java/dev/ltms/fleet/Fleetd.java @@ -134,6 +134,10 @@ public final class Fleetd { // secret is reported above, so upgrading past this commit never silently drops CB-592's // protection. reportMemberCredentialsGap(cfg); + // fleetd #395: an unset exhaustedPattern is a silent opt-out of usage-limit detection for + // that profile — say so loudly, the same way the two reports above do, rather than let an + // operator discover it only when a limit goes undetected. + reportExhaustedPatternGap(cfg); // CB-559: `cfg` stays the startup snapshot — every validation and every piece of one-time // wiring below reads it, and must, because those decisions cannot be unmade. `config` is the // live reference the hot paths read per use. Which keys can actually move is ConfigRef's @@ -664,7 +668,13 @@ public final class Fleetd { FleetMcp.QuarantineSource quarantineSource = new FleetMcp.QuarantineSource(profile -> { var configured = config.get().profiles().get(profile); return configured == null ? null : configured.effectiveCredentialId(); - }, quarantine); + }, quarantine, profile -> { + // fleetd #395: read live off the current config, like credentialIdFor above — an + // exhaustedPattern edit takes effect on the next fleet_profiles/fleet_list call, no + // restart needed, same as the credential-id lookup it sits beside. + var configured = config.get().profiles().get(profile); + return configured != null && configured.hasExhaustedPattern(); + }); FleetMcp.OutageSource outageSource = new FleetMcp.OutageSource(profile -> { var configured = config.get().profiles().get(profile); return configured == null ? null : configured.effectiveCredentialId(); @@ -1323,6 +1333,52 @@ public final class Fleetd { + "fleetd.yaml — see fleetd.example.yaml — and restart."); } + /** + * fleetd #395: {@code exhaustedPattern} (see {@link FleetConfig.Profile#exhaustedPattern}) is + * deliberately opt-in — {@code null}/blank means a backend refusal on that profile is never + * classified as {@code BACKEND_EXHAUSTED}, so its credential is never quarantined. That is a + * legitimate choice (guessing the vendor's wording would be worse), but an operator who never + * opted a profile in should not discover the gap only when a usage limit silently goes + * undetected. Warn once at startup, naming every unarmed profile, exactly like {@link + * #reportMemberCredentialsGap} — never refuse to start over it. + * + *

A {@code subscription: true} profile that is unarmed gets a SECOND, louder WARN of its + * own: it bills the operator's metered Claude plan, the case where an undetected usage limit + * costs the most. + * + *

Package-private so a test can capture the real log via a {@link + * ch.qos.logback.core.read.ListAppender}, the same pattern {@link + * #reportMemberCredentialsGap}'s own test uses. + */ + static void reportExhaustedPatternGap(FleetConfig cfg) { + List unarmedSubscription = new ArrayList<>(); + List unarmedOther = new ArrayList<>(); + cfg.profiles().forEach((name, profile) -> { + if (!profile.hasExhaustedPattern()) { + (profile.isSubscription() ? unarmedSubscription : unarmedOther).add(name); + } + }); + if (unarmedSubscription.isEmpty() && unarmedOther.isEmpty()) { + log.info("exhaustedPattern: every configured profile has usage-limit detection armed"); + return; + } + List allUnarmed = new ArrayList<>(unarmedSubscription); + allUnarmed.addAll(unarmedOther); + allUnarmed = allUnarmed.stream().sorted().toList(); + log.warn("exhaustedPattern: profile(s) {} have no exhaustedPattern configured — a " + + "usage-limit refusal on any of them is never detected and never " + + "quarantines its credential. Set exhaustedPattern (see " + + "fleetd.example.yaml) to arm detection for a profile.", + allUnarmed); + if (!unarmedSubscription.isEmpty()) { + List sortedSubscription = unarmedSubscription.stream().sorted().toList(); + log.warn("exhaustedPattern: subscription profile(s) {} run on the operator's metered " + + "Claude plan and have NO usage-limit detection armed — this is the " + + "case where a missed usage limit costs the most.", + sortedSubscription); + } + } + /** * Poll herdr's {@code ping} until it answers or {@link #HERDR_WAIT_SECONDS} elapses (CB-504). * diff --git a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java index 84a28ca..53125b8 100644 --- a/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java +++ b/fleetd/src/main/java/dev/ltms/fleet/mcp/FleetMcp.java @@ -119,9 +119,28 @@ public final class FleetMcp { /** * CB-578 stage B quarantine facts used by {@code fleet_profiles}: a profile → credential id * lookup, plus the shared {@link BackendQuarantine} to read remaining cooldowns off. + * + * @param exhaustedPatternArmed fleetd #395: profile → whether that profile's {@code + * exhaustedPattern} is configured (see {@code + * FleetConfig.Profile#hasExhaustedPattern}), i.e. whether a backend refusal + * on it can EVER be classified {@code BACKEND_EXHAUSTED} and quarantine its + * credential. Bundled here, not a separate Source, because it answers the + * exact question {@code fleet_profiles}'s quarantine facts already answer + * for a QUARANTINED profile — "can this profile's usage limit ever be + * caught?" — just for every profile, not only one currently caught. */ - public record QuarantineSource(Function credentialIdFor, BackendQuarantine quarantine) { - /** Inert source — no profile is ever reported quarantined. Explicit stand-in, not a default. */ + public record QuarantineSource(Function credentialIdFor, BackendQuarantine quarantine, + Function exhaustedPatternArmed) { + /** + * Backward-compatible 2-arg form, before fleetd #395 added {@code exhaustedPatternArmed} — + * reports every profile unarmed. Keeps every pre-existing call site (production and test) + * compiling and behaving identically for the quarantine facts they actually asked for. + */ + public QuarantineSource(Function credentialIdFor, BackendQuarantine quarantine) { + this(credentialIdFor, quarantine, _ -> false); + } + + /** Inert source — no profile is ever reported quarantined or armed. Explicit stand-in, not a default. */ public static QuarantineSource none() { return new QuarantineSource(_ -> null, BackendQuarantine.none()); } } @@ -1109,6 +1128,14 @@ public final class FleetMcp { * not the other, and the two then disagree about a live outage. That is exactly what fleetd * #284 was, where one rule computed in two places was widened in only one and a single response * contradicted itself. Shared inputs do not make duplicated computation safe. + * + *

fleetd #395: also reports {@code exhaustionDetectionArmed}, one boolean per configured + * profile — {@code true} when that profile's {@code exhaustedPattern} is set, {@code false} + * when it is not, so an operator can tell "this profile is healthy" from "nothing can ever + * quarantine this profile" without reading {@code fleetd.yaml}. Unlike {@code quarantined}/ + * {@code coolingOff}, this map always names every profile: an unarmed profile never enters a + * transient state to be absent from, so silence here would read as "healthy" rather than "not + * being watched at all". */ public static Map profilesView(PeerLauncher workers, QuarantineSource quarantine, OutageSource outage) { Map result = new LinkedHashMap<>(); @@ -1116,7 +1143,9 @@ public final class FleetMcp { result.put("default", workers.defaultProfile() == null ? "" : workers.defaultProfile()); Map quarantined = new LinkedHashMap<>(); Map coolingOff = new LinkedHashMap<>(); + Map exhaustionDetectionArmed = new LinkedHashMap<>(); for (String profile : workers.profiles()) { + exhaustionDetectionArmed.put(profile, quarantine.exhaustedPatternArmed().apply(profile)); String credentialId = quarantine.credentialIdFor().apply(profile); if (credentialId != null) { quarantine.quarantine().remainingSeconds(credentialId).ifPresent(remaining -> { @@ -1136,6 +1165,7 @@ public final class FleetMcp { }); } } + result.put("exhaustionDetectionArmed", exhaustionDetectionArmed); if (!quarantined.isEmpty()) { result.put("quarantined", quarantined); } @@ -1663,7 +1693,11 @@ public final class FleetMcp { "List the configured worker profiles (backends) and which one fleet_spawn uses by " + "default. A 'quarantined' map is present when a backend-exhausted refusal put " + "a profile's credential on cooldown — fleet_spawn onto it is refused until " - + "quarantinedForSeconds elapses; a profile sharing that credential is listed too.", + + "quarantinedForSeconds elapses; a profile sharing that credential is listed too. " + + "'exhaustionDetectionArmed' reports, per profile, whether a usage-limit refusal " + + "on it can EVER be classified and quarantined (its exhaustedPattern is " + + "configured) — false means that profile's credential can never be quarantined " + + "by this mechanism, however many usage-limit refusals it sees.", objectSchema(Map.of(), List.of())); } diff --git a/fleetd/src/test/java/dev/ltms/fleet/ExhaustedPatternGapReportTest.java b/fleetd/src/test/java/dev/ltms/fleet/ExhaustedPatternGapReportTest.java new file mode 100644 index 0000000..bdc4fb4 --- /dev/null +++ b/fleetd/src/test/java/dev/ltms/fleet/ExhaustedPatternGapReportTest.java @@ -0,0 +1,150 @@ +package dev.ltms.fleet; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; +import dev.ltms.fleet.config.FleetConfig; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.LoggerFactory; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * fleetd #395: {@code exhaustedPattern} (see {@link FleetConfig.Profile#exhaustedPattern}) is + * deliberately opt-in — an unset one leaves usage-limit detection silently OFF for that profile, + * and nothing quarantines its credential. {@link Fleetd#reportExhaustedPatternGap} must say so at + * startup, naming every unarmed profile, and must never fire when every profile is armed. Mirrors + * {@link MemberCredentialsGapReportTest}'s pattern, capturing the real log via a + * {@link ListAppender}. + * + *

The 8-profile shape in {@link #theLiveEightProfileShapeWarnsExactlyTheSixUnarmedProfiles} is + * the live {@code fleetd.yaml} shape measured 2026-09-10 (fleetd #395's own ticket): 6 of 8 + * profiles unarmed, 2 of those 6 ({@code opus}, {@code sonnet}) running on the operator's Claude + * subscription. {@code fleetd.yaml} itself is gitignored and unavailable to this test, so the + * shape is reproduced as a throwaway config in a {@code @TempDir} rather than read off disk. + */ +class ExhaustedPatternGapReportTest { + + private static FleetConfig load(Path dir, String yaml) throws Exception { + Path f = dir.resolve("fleetd.yaml"); + Files.writeString(f, yaml); + return FleetConfig.load(f); + } + + private static ListAppender attach() { + Logger logger = (Logger) LoggerFactory.getLogger(Fleetd.class); + ListAppender appender = new ListAppender<>(); + appender.start(); + logger.addAppender(appender); + return appender; + } + + private static void detach(ListAppender appender) { + ((Logger) LoggerFactory.getLogger(Fleetd.class)).detachAppender(appender); + } + + /** Every profile name mentioned by a WARN-level log line, across every WARN this call produced. */ + private static List warnMessages(ListAppender appender) { + return appender.list.stream() + .filter(e -> e.getLevel() == Level.WARN) + .map(ILoggingEvent::getFormattedMessage) + .toList(); + } + + @Test + void theLiveEightProfileShapeWarnsExactlyTheSixUnarmedProfiles(@TempDir Path dir) throws Exception { + // Reproduces the live shape measured 2026-09-10: 8 profiles, 2 armed (sol, terra), 6 + // unarmed (local, local-direct, gx, opus, sonnet, xf) — 2 of the unarmed 6 (opus, sonnet) + // are subscription: true. + FleetConfig cfg = load(dir, """ + profiles: + local: + baseUrl: http://gx00.gw:8000 + local-direct: + baseUrl: http://gx01.gw:8000 + gx: + kind: opencode + baseUrl: https://llm.ltms.dev/v1 + opus: + subscription: true + model: claude-opus-5 + sonnet: + subscription: true + model: claude-sonnet-5 + sol: + baseUrl: https://llm.ltms.dev/v1 + exhaustedPattern: "The usage limit has been reached" + terra: + baseUrl: https://llm.ltms.dev/v1 + exhaustedPattern: "The usage limit has been reached" + xf: + baseUrl: https://llm.ltms.dev/v1 + """); + + ListAppender appender = attach(); + try { + Fleetd.reportExhaustedPatternGap(cfg); + } finally { + detach(appender); + } + + List warns = warnMessages(appender); + assertFalse(warns.isEmpty(), "6 of 8 profiles are unarmed — at least one WARN must fire"); + + // Exactly one WARN aggregates every unarmed profile, naming all 6 and none of the 2 armed. + String aggregate = warns.stream() + .filter(m -> m.contains("no exhaustedPattern configured")) + .findFirst() + .orElseThrow(() -> new AssertionError("expected an aggregate unarmed-profiles WARN: " + warns)); + for (String unarmed : List.of("local", "local-direct", "gx", "opus", "sonnet", "xf")) { + assertTrue(aggregate.contains(unarmed), "aggregate WARN must name '" + unarmed + "': " + aggregate); + } + for (String armed : List.of("sol", "terra")) { + assertFalse(aggregate.contains(armed), "aggregate WARN must NOT name armed profile '" + armed + "': " + aggregate); + } + + // A second, louder WARN calls out the subscription profiles specifically. + String subscriptionWarn = warns.stream() + .filter(m -> m.contains("metered Claude plan")) + .findFirst() + .orElseThrow(() -> new AssertionError("expected a subscription-specific WARN: " + warns)); + assertTrue(subscriptionWarn.contains("opus"), subscriptionWarn); + assertTrue(subscriptionWarn.contains("sonnet"), subscriptionWarn); + assertFalse(subscriptionWarn.contains("local-direct"), + "the subscription WARN must not name a non-subscription profile: " + subscriptionWarn); + } + + @Test + void everyProfileArmedProducesNoWarningAtAll(@TempDir Path dir) throws Exception { + FleetConfig cfg = load(dir, """ + profiles: + sol: + baseUrl: https://llm.ltms.dev/v1 + exhaustedPattern: "The usage limit has been reached" + terra: + baseUrl: https://llm.ltms.dev/v1 + exhaustedPattern: "The usage limit has been reached" + opus: + subscription: true + model: claude-opus-5 + exhaustedPattern: "5-hour limit reached" + """); + + ListAppender appender = attach(); + try { + Fleetd.reportExhaustedPatternGap(cfg); + } finally { + detach(appender); + } + + assertTrue(warnMessages(appender).isEmpty(), + "every profile is armed — a checker that warns anyway always fires: " + warnMessages(appender)); + } +} diff --git a/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetProfilesArmedFieldTest.java b/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetProfilesArmedFieldTest.java new file mode 100644 index 0000000..043251f --- /dev/null +++ b/fleetd/src/test/java/dev/ltms/fleet/mcp/FleetProfilesArmedFieldTest.java @@ -0,0 +1,62 @@ +package dev.ltms.fleet.mcp; + +import dev.ltms.fleet.config.FleetConfig; +import dev.ltms.fleet.guard.SubscriptionGuard; +import dev.ltms.fleet.herdr.AgentControl; +import dev.ltms.fleet.herdr.FakeHerdr; +import dev.ltms.fleet.herdr.WorkspaceControl; +import dev.ltms.fleet.member.ClaudeCodeLauncher; +import dev.ltms.fleet.peer.PeerLauncher; +import io.modelcontextprotocol.spec.McpSchema; +import org.junit.jupiter.api.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * fleetd #395: {@code fleet_profiles} must let an operator tell "this profile's usage-limit + * detection is armed" from "nothing can ever quarantine this profile" — see {@link + * FleetMcp.QuarantineSource#exhaustedPatternArmed()} and {@link + * FleetMcp#profilesView(PeerLauncher, FleetMcp.QuarantineSource, FleetMcp.OutageSource)}. + */ +class FleetProfilesArmedFieldTest { + + private static PeerLauncher twoProfileLauncher(FakeHerdr h) { + FleetConfig.Profile armed = new FleetConfig.Profile( + "armed-profile", "http://gx00.gw:8000", "coder", null, "FLEETD_WORKER_TOKEN", null, + "tab", "fleetd-workers", "worker: {profile} #{n}", null, null, null); + FleetConfig.Profile unarmed = new FleetConfig.Profile( + "unarmed-profile", "http://gx01.gw:8000", "coder", null, "FLEETD_WORKER_TOKEN", null, + "tab", "fleetd-workers", "worker: {profile} #{n}", null, null, null); + Map profiles = new LinkedHashMap<>(); + profiles.put(armed.profile(), armed); + profiles.put(unarmed.profile(), unarmed); + return new ClaudeCodeLauncher(new AgentControl(h), new WorkspaceControl(h), + new SubscriptionGuard(Set.of("gx00.gw", "gx01.gw")), profiles, armed.profile(), _ -> "tok"); + } + + private static String textOf(McpSchema.CallToolResult r) { + return ((McpSchema.TextContent) r.content().getFirst()).text(); + } + + @Test + void armedProfileReportsArmedAndUnarmedReportsUnarmed() { + FakeHerdr h = new FakeHerdr(); + PeerLauncher workers = twoProfileLauncher(h); + FleetMcp.QuarantineSource source = new FleetMcp.QuarantineSource( + _ -> null, dev.ltms.fleet.placement.BackendQuarantine.none(), + profile -> "armed-profile".equals(profile)); + + McpSchema.CallToolResult res = FleetMcp.profiles(workers, source); + assertNotEquals(Boolean.TRUE, res.isError()); + String out = textOf(res); + + assertTrue(out.contains("\"exhaustionDetectionArmed\""), out); + assertTrue(out.contains("\"armed-profile\":true"), out); + assertTrue(out.contains("\"unarmed-profile\":false"), out); + } +}