Merge CB-551: idle-lead heartbeat
CI / build (push) Successful in 1m15s
CI / contract (push) Successful in 1m21s

This commit is contained in:
Dai Ha
2026-08-13 21:21:33 +02:00
9 changed files with 729 additions and 2 deletions
+20
View File
@@ -84,6 +84,26 @@ bind:
# tabPrefix: "lead:" # `lead: opus-5.0` ⇒ a lead named opus-5.0 (case-insensitive; default "lead:")
# intervalSeconds: 10 # rescan cadence, and the worst case before a new tab is recognised
# CB-551: IDLE-LEAD HEARTBEAT — nudge the single lead back to work when it has been continuously
# idle (no open bridge_send driving it) past the quiet period. The fleet is one lead + architects +
# workers, so a lead that stalls is a single point of failure; the ReplyPushLoop only nudges when a
# reply lands, and this timer catches the gap where nothing lands and the lead just sits idle.
#
# Opt-in on purpose — it SPENDS the operator's subscription on its own initiative (each nudge starts
# a lead turn nobody asked for), so upgrading the daemon must never switch it on for you. Absent
# block = feature off, exactly as before.
#
# Three knobs, each with a default that errs on the side of not burning context:
# idleAfterSeconds: 300 # how long the lead must stay idle before the FIRST nudge (default 300 —
# # absorbs normal post-turn pauses; re-prompting every pause burns context)
# backoffMs: 60000 # re-check cadence / spacing between nudges past the quiet period (default 60000)
# quietNudgeCap: 3 # cap on consecutive nudges that find NOTHING pending, then it stops
# # until real state appears (default 3 — never nag an empty fleet forever)
# leadHeartbeat:
# idleAfterSeconds: 300
# backoffMs: 60000
# quietNudgeCap: 3
# herdr Unix socket. Omit to use the client default
# (${HERDR_SOCKET_PATH:-~/.config/herdr/herdr.sock}).
herdrSocket: ~/.config/herdr/herdr.sock
@@ -28,6 +28,7 @@ import dev.ltms.bridged.msg.InMemoryReplyInbox;
import dev.ltms.bridged.msg.MessageService;
import dev.ltms.bridged.msg.Rendezvous;
import dev.ltms.bridged.msg.ReplyInbox;
import dev.ltms.bridged.msg.LeadHeartbeatLoop;
import dev.ltms.bridged.msg.ReplyPushLoop;
import dev.ltms.bridged.rest.BridgedApp;
import dev.ltms.bridged.session.GitWorktrees;
@@ -296,6 +297,23 @@ public final class Bridged {
Metrics metrics = BridgedMetrics.create(sessions, replyInbox);
var pushLoop = new ReplyPushLoop(primaryRegistry, agents, replyInbox,
pushScheduler, maxReminders, backoffMs, metrics);
// CB-551: idle-lead heartbeat. Opt-in; absent `leadHeartbeat:` this is never constructed, so
// an upgraded daemon cannot silently start spending subscription on nudging an idle lead.
// It has its own single-thread scheduler and holds its own scheduler shutdown via close().
final LeadHeartbeatLoop heartbeat;
var heartbeatScheduler = Executors.newSingleThreadScheduledExecutor(r ->
Thread.ofVirtual().name("bridge-heartbeat-").unstarted(r));
if (cfg.leadHeartbeat() != null) {
var hb = cfg.leadHeartbeat();
heartbeat = new LeadHeartbeatLoop(primaryRegistry, agents, replyInbox, sessions::roster,
pushLoop, heartbeatScheduler, System::nanoTime,
TimeUnit.SECONDS.toNanos(hb.idleAfterSeconds()), hb.backoffMs(), hb.quietNudgeCap(),
metrics);
heartbeat.start();
} else {
heartbeat = null;
heartbeatScheduler.shutdownNow();
}
MessageService messages = new MessageService(agents, injector, rendezvous, replyInbox,
pushLoop, metrics);
@@ -346,6 +364,7 @@ public final class Bridged {
poller.stop();
messages.close();
pushLoop.close();
if (heartbeat != null) heartbeat.close(); // CB-551: stop the idle-lead heartbeat scheduler
mcp.close();
if (reaper != null) reaper.stop();
// Release the broker connection last among message resources (no-op for the in-memory inbox).
@@ -53,6 +53,8 @@ import java.util.Set;
* to a slot. Nothing here spawns a slot.
* @param leadScan opt-in discovery of leads by tab label (CB-531); {@code null} ⇒ no scanning,
* and only {@code leaders:}/{@code primary:} name a lead
* @param leadHeartbeat opt-in idle-lead heartbeat (CB-551); {@code null} ⇒ off, and an upgraded
* daemon never nudges an idle lead on its own initiative
* @param placement how to choose a worker profile for an unqualified spawn:
* {@code fixed} (default), {@code round-robin}, or {@code weighted}
* @param auth API authentication mode ({@code null} → {@code loopback-trust}, the
@@ -75,6 +77,7 @@ public record BridgedConfig(
Map<String, Leader> leaders,
Map<String, Architect> architects,
LeadScan leadScan,
LeadHeartbeat leadHeartbeat,
String placement,
Auth auth) {
@@ -441,6 +444,40 @@ public record BridgedConfig(
}
}
/**
* Opt-in idle-lead heartbeat (CB-551): when the single lead has been continuously idle past
* {@code idleAfterSeconds} with no open {@code bridge_send} driving it, nudge it back to work.
*
* <p>Deliberately opt-in ({@code null} ⇒ off, exactly like {@code leadScan:}). The heartbeat
* spends the operator's model subscription on its own initiative — it prompts the lead to start
* a turn nobody asked for — which is precisely the kind of behaviour a daemon upgrade must never
* acquire silently. Absent the block, nothing is constructed and nothing fires.
*
* @param idleAfterSeconds how long the lead must be <em>continuously</em> injectable before the
* first nudge (default 300). Why: a lead that just finished a turn sits
* momentarily idle; re-prompting instantly would turn every natural pause
* into a new turn and burn the lead's context for nothing. Context
* exhaustion is what actually kills a long-running lead, so every nudge
* must earn its cost — 5 minutes absorbs normal pauses without stalling.
* @param backoffMs re-check cadence once the quiet period has passed (default 60000). The
* loop samples on this interval; 60s is gentle enough not to burn context
* while prompt enough to unstick a stalled lead promptly. Also the spacing
* between nudges to a lead that is idle past the quiet period.
* @param quietNudgeCap cap on <em>consecutive</em> nudges that find no pending fleet state
* (default 3), before the loop stops until real state appears again. Why:
* each such nudge costs the lead a turn just to read "nothing pending";
* three is enough to tell it it may stand down without nagging forever, and
* it is the bound that stops an idle fleet from being a subscription burner.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public record LeadHeartbeat(Integer idleAfterSeconds, Long backoffMs, Integer quietNudgeCap) {
public LeadHeartbeat {
idleAfterSeconds = (idleAfterSeconds == null || idleAfterSeconds <= 0) ? 300 : idleAfterSeconds;
backoffMs = (backoffMs == null || backoffMs <= 0) ? 60_000L : backoffMs;
quietNudgeCap = (quietNudgeCap == null || quietNudgeCap < 0) ? 3 : quietNudgeCap;
}
}
/**
* The terminal → lead-name map that {@link dev.ltms.bridged.auth.CallerResolver} resolves
* against, merging the {@code leaders:} registry with the legacy singular {@code primary:} pin.
@@ -571,7 +608,7 @@ public record BridgedConfig(
private static final Set<String> KNOWN_TOP_LEVEL_KEYS = Set.of(
"bind", "herdrSocket", "worker", "workers", "defaultWorker", "guard", "worktreeRoot",
"lifecycle", "spawnReadyTimeoutMs", "spawnReadyPollMs", "broker", "primary", "leaders",
"architects", "leadScan", "placement", "auth");
"architects", "leadScan", "leadHeartbeat", "placement", "auth");
/** Load and validate config from {@code path}. */
public static BridgedConfig load(Path path) {
@@ -743,9 +780,11 @@ public record BridgedConfig(
// leadScan is left as-is: null is "off", and LeadScan's own compact constructor defaults the
// fields of a block that IS present. Defaulting it here would switch the feature on for
// every config that never mentioned it.
// leadHeartbeat is left as-is for the same reason (CB-551): null is "off", and LeadHeartbeat's
// own compact constructor defaults the fields of a block that IS present.
// architects is left as-is: null is "none configured", and Architect's fields have no
// defaults to fill. Defaulting it here would change nothing, so leave the call natural.
return new BridgedConfig(b, herdrSocket, worker, workers, defaultWorker, g, worktreeRoot, l, timeout, pollMs, broker, primary, leaders, architects, leadScan, placementOrDefault, a);
return new BridgedConfig(b, herdrSocket, worker, workers, defaultWorker, g, worktreeRoot, l, timeout, pollMs, broker, primary, leaders, architects, leadScan, leadHeartbeat, placementOrDefault, a);
}
/**
@@ -26,6 +26,8 @@ public final class BridgedMetrics {
public static final String REPLIES = "bridged_replies_total";
/** Counter: push-loop nudges to the primary, by outcome. */
public static final String PUSH_NUDGES = "bridged_push_nudges_total";
/** Counter: idle-lead heartbeat nudges to the lead, by outcome (CB-551). */
public static final String HEARTBEAT_NUDGES = "bridged_lead_heartbeat_nudges_total";
/** Counter: spawn attempts by peer kind and outcome. */
public static final String SPAWNS = "bridged_spawns_total";
/** Counter: herdr socket calls by method and outcome. */
@@ -55,6 +57,9 @@ public final class BridgedMetrics {
"Worker replies by delivery path (rendezvous=resolved an open send, inbox=stranded and held).");
m.describe(PUSH_NUDGES, "counter",
"CB-307 push-loop nudges to the primary (delivered|exhausted).");
m.describe(HEARTBEAT_NUDGES, "counter",
"CB-551 idle-lead heartbeat nudges (delivered|failed|exhausted). Quiet-cap exhaustion "
+ "means the lead idled with nothing pending and was told to stand down.");
m.describe(SPAWNS, "counter",
"Worker spawn attempts by peer kind and outcome (ready|timeout|guard_rejected).");
m.describe(HERDR_CALLS, "counter",
@@ -0,0 +1,351 @@
package dev.ltms.bridged.msg;
import dev.ltms.bridged.herdr.AgentControl;
import dev.ltms.bridged.herdr.AgentStatus;
import dev.ltms.bridged.mcp.PrimaryRegistry;
import dev.ltms.bridged.metrics.BridgedMetrics;
import dev.ltms.bridged.metrics.Metrics;
import dev.ltms.bridged.session.WorkerSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
/**
* CB-551: an opt-in heartbeat that nudges the single idle lead back to work once it has been
* continuously idle past a quiet period with no open {@code bridge_send} driving it.
*
* <p>Why this exists: the fleet is ONE lead + architects + workers, so an idle, stalled lead is a
* single point of failure for the fleet's progress. {@link ReplyPushLoop} nudges the lead only when
* a worker reply lands; this loop is the timer that catches the gap where nothing lands and the
* lead simply sits idle with nobody to prompt it onward.
*
* <p>Mechanism mirrors {@link ReplyPushLoop}: a pure, unit-testable decision function
* ({@link #decide(long, Long, int, AgentStatus, boolean, boolean, FleetState)}) plus a thin
* scheduler around it. The loop is <em>entirely opt-in</em> ({@code leadHeartbeat:} in config); with
* no such block it is never constructed, so upgrading the daemon cannot silently acquire a behaviour
* that spends the operator's model subscription on its own initiative (constraint 1).
*
* <p>Four invariants keep it from becoming a runaway subscription burner:
* <ol>
* <li><b>Status-gated</b> — a {@code WORKING} lead is making progress and is never touched; only an
* injectable (idle/done/blocked) lead is even considered (constraint 2).</li>
* <li><b>Debounced</b> — a nudge only fires once the lead has been <em>continuously</em> idle past
* {@code idleAfterSeconds}, so a lead that just finished a turn is not re-prompted into every
* natural pause (constraint 3).</li>
* <li><b>Bounded when quiet</b> — consecutive nudges that find no pending fleet state are capped at
* {@code quietNudgeCap}, then the loop stops nudging until real state returns (constraint 4).</li>
* <li><b>Never races {@link ReplyPushLoop}</b> — while that loop is actively nudging any target this
* loop stands down, so two competing injections never start two turns in the same pane
* (constraint 6).</li>
* </ol>
*/
public final class LeadHeartbeatLoop {
private static final Logger log = LoggerFactory.getLogger(LeadHeartbeatLoop.class);
/** Sentinel for {@link #idleSinceNanos}: the lead is not currently in an idle stretch. */
private static final long NOT_IDLE = Long.MIN_VALUE;
private final PrimaryRegistry primaryRegistry;
private final AgentControl agents;
private final ReplyInbox inbox;
private final Supplier<List<WorkerSession>> roster;
private final ReplyPushLoop pushLoop;
private final ScheduledExecutorService scheduler;
private final LongSupplier clock;
private final long idleAfterNanos;
private final long backoffMs;
private final int quietNudgeCap;
private final Metrics metrics; // CB-512 pattern: nullable — no registry in unit tests
/** When the current idle stretch began (nanos), or {@link #NOT_IDLE}. Single scheduler thread only. */
private long idleSinceNanos = NOT_IDLE;
/** Consecutive nudges that found no pending fleet state. Single scheduler thread only. */
private int quietCount = 0;
/** Constructor with an injectable clock and no metric registry (unit tests, or wiring that opts out). */
public LeadHeartbeatLoop(PrimaryRegistry primaryRegistry, AgentControl agents, ReplyInbox inbox,
Supplier<List<WorkerSession>> roster, ReplyPushLoop pushLoop,
ScheduledExecutorService scheduler, LongSupplier clock,
long idleAfterNanos, long backoffMs, int quietNudgeCap) {
this(primaryRegistry, agents, inbox, roster, pushLoop, scheduler, clock,
idleAfterNanos, backoffMs, quietNudgeCap, null);
}
/** As above, with a metric registry (the CB-512 pattern) so nudge outcomes are counted. */
public LeadHeartbeatLoop(PrimaryRegistry primaryRegistry, AgentControl agents, ReplyInbox inbox,
Supplier<List<WorkerSession>> roster, ReplyPushLoop pushLoop,
ScheduledExecutorService scheduler, LongSupplier clock,
long idleAfterNanos, long backoffMs, int quietNudgeCap, Metrics metrics) {
this.primaryRegistry = primaryRegistry;
this.agents = agents;
this.inbox = inbox;
this.roster = roster;
this.pushLoop = pushLoop;
this.scheduler = scheduler;
this.clock = clock;
this.idleAfterNanos = idleAfterNanos;
this.backoffMs = backoffMs;
this.quietNudgeCap = quietNudgeCap;
this.metrics = metrics;
}
/** Count one nudge outcome when a registry is wired; a no-op in unit tests. */
private void countNudge(String outcome) {
if (metrics != null) {
metrics.inc(BridgedMetrics.HEARTBEAT_NUDGES, "outcome", outcome);
}
}
// --- decision logic (package-private for unit-testing) --------------------------------------
/** The action the loop should take on one tick. */
enum Action {
/** Send a heartbeat nudge to the lead now. */
INJECT,
/** Lead is idle but not yet past the quiet period — keep waiting, do nothing. */
WAIT_IDLE,
/** Lead is making progress (or its status cannot be read) — reset the idle window and re-arm. */
LEAD_BUSY,
/** Idle past the quiet period with nothing pending and the cap exhausted — stop until new state appears. */
QUIET_DONE,
/** {@link ReplyPushLoop} is actively nudging — stand aside rather than start a competing turn. */
STAND_DOWN
}
/** The outcome of one decision: the action plus the state to persist for the next tick. */
record Decision(Action action, Long idleSinceNanos, int quietCount) {}
/**
* Pure decision function: given the current loop state and fleet/lead facts, return what to do
* next and the exact state to carry forward. Pure means no I/O and no mutation — the caller
* ({@link #tick}) applies {@link Decision} to its fields. This is what makes the loop unit-testable
* with a fake clock and no sleeping.
*
* @param nowNanos the current time (an injected clock in tests, {@link System#nanoTime()} live)
* @param idleSinceNanos when the current idle stretch began, or {@code null} if the lead is not idle
* @param quietCount consecutive nudges so far that found no pending fleet state
* @param status the lead's herdr status
* @param pushLoopActive whether {@link ReplyPushLoop} is currently nudging some target (constraint 6)
* @param leadKnown whether a lead terminal is known to nudge at all
* @param fleet a snapshot of the pending fleet state (constraint 5)
* @return the action to take and the state to persist
*/
Decision decide(long nowNanos, Long idleSinceNanos, int quietCount, AgentStatus status,
boolean pushLoopActive, boolean leadKnown, FleetState fleet) {
// Constraint 6: while ReplyPushLoop is actively nudging the lead, injecting a second,
// competing prompt into the same pane would start a second turn — racing loops multiply
// turns and context burn. Stand aside, and treat the active push as real state (re-arm the
// quiet counter), because the reply that drove it is exactly the kind of new state that
// should reset the cap.
if (pushLoopActive) {
return new Decision(Action.STAND_DOWN, idleSinceNanos, 0);
}
// Constraint 2: a WORKING lead is making progress and must NOT be touched; an unreadable
// status (read failure, or the agent is gone) is safest treated the same way — never inject
// into a state we cannot read. Either way, reset the idle window and the quiet counter: the
// lead was / may be active, so the next idle stretch must count its own quiet period fresh.
if (status == null || !status.injectable()) {
return new Decision(Action.LEAD_BUSY, null, 0);
}
if (idleSinceNanos == null) {
// The lead just became injectable — record the start of an idle stretch and wait out the
// debounce quiet period before ever nudging (constraint 3).
return new Decision(Action.WAIT_IDLE, nowNanos, quietCount);
}
if (nowNanos - idleSinceNanos < idleAfterNanos) {
// Still within the quiet period: the lead that just finished a turn sits momentarily idle
// and must not be re-prompted into every natural pause.
return new Decision(Action.WAIT_IDLE, idleSinceNanos, quietCount);
}
// Past the quiet period with an injectable lead: it is a genuine candidate for a nudge. Two
// gating facts decide whether and how:
if (!leadKnown) {
// No lead terminal is known yet (e.g. the registry has not learned one) — there is nobody
// to nudge. Keep waiting; the window stays open so discovery re-arms it without a fresh
// quiet period.
return new Decision(Action.WAIT_IDLE, idleSinceNanos, quietCount);
}
if (fleet.hasPending()) {
// Real fleet state is waiting — a worker reply or a DONE session. This is new state, so
// it resets the quiet counter (constraint 4) and the lead is nudged to go collect it.
return new Decision(Action.INJECT, idleSinceNanos, 0);
}
if (quietCount < quietNudgeCap) {
// Nothing is pending, but the cap is not exhausted: nudge anyway, telling the lead
// exactly that nothing is waiting so it can choose to stand down rather than hunt
// (constraint 5). Count it toward the consecutive-quiet cap.
return new Decision(Action.INJECT, idleSinceNanos, quietCount + 1);
}
// Nothing pending and the cap is exhausted: stop nudging until real state appears again
// (constraint 4). The loop still ticks on backoff so a genuinely new reply or session change
// re-arms it — QUIET_DONE stops injection, not observation.
return new Decision(Action.QUIET_DONE, idleSinceNanos, quietCount);
}
// --- loop ----------------------------------------------------------------------------------
/**
* Start the heartbeat. Schedules the first tick after one backoff so a freshly-booting daemon
* does not evaluate the lead's idle state before the fleet has settled.
*/
public void start() {
log.info("idle-lead heartbeat: on — nudge lead after {}s idle (recheck {}ms, quiet cap {})",
TimeUnit.NANOSECONDS.toSeconds(idleAfterNanos), backoffMs, quietNudgeCap);
scheduler.schedule(this::tick, backoffMs, TimeUnit.MILLISECONDS);
}
/** One loop tick, every {@link #backoffMs} — the thin scheduler around {@link #decide}. */
private void tick() {
boolean leadKnown = primaryRegistry.primaryTerminal().isPresent();
FleetState fleet = snapshot(inbox, roster);
AgentStatus status = AgentStatus.UNKNOWN;
if (leadKnown) {
try {
status = agents.status(primaryRegistry.primaryTerminal().orElseThrow());
} catch (RuntimeException e) {
// A failed status read degrades to "unknown" — decide() treats that like a busy lead
// and never injects into a state it cannot read. Retry on the next backoff.
log.debug("idle-heartbeat: status check failed for lead, will retry: {}", e.toString());
}
}
Decision d = decide(clock.getAsLong(),
idleSinceNanos == NOT_IDLE ? null : idleSinceNanos,
quietCount, status, pushLoop.isActive(), leadKnown, fleet);
applyDecision(d);
switch (d.action()) {
case INJECT -> injectNudge(fleet);
case QUIET_DONE -> countNudge("exhausted");
case WAIT_IDLE, LEAD_BUSY, STAND_DOWN -> { /* nothing to inject, nothing to count */ }
}
scheduleNext();
}
/** Persist the state a decision returned, so the next tick starts from it. */
private void applyDecision(Decision d) {
idleSinceNanos = d.idleSinceNanos() == null ? NOT_IDLE : d.idleSinceNanos();
quietCount = d.quietCount();
}
/** Send the nudge to the known lead. */
private void injectNudge(FleetState fleet) {
var lead = primaryRegistry.primaryTerminal();
if (lead.isEmpty()) {
return; // the lead disappeared between the decision and the injection
}
String leadTerminal = lead.get();
try {
agents.send(leadTerminal, fleet.nudgeText());
log.debug("idle-heartbeat: nudge sent to lead {} (quiet nudges so far in this stretch: {})",
leadTerminal, quietCount);
countNudge("delivered");
} catch (RuntimeException e) {
log.warn("idle-heartbeat: failed to nudge lead {}: {}", leadTerminal, e.toString());
countNudge("failed");
}
}
/** Schedule the next tick on the scheduler thread pool. */
private void scheduleNext() {
scheduler.schedule(this::tick, backoffMs, TimeUnit.MILLISECONDS);
}
// --- lifecycle -----------------------------------------------------------------------------
/** Shut down the scheduler; outstanding ticks are cancelled. */
public void stop() {
scheduler.shutdownNow();
}
/** @see #stop() */
public void close() {
stop();
}
/**
* A read-only snapshot of the fleet facts the heartbeat reports to an idle lead. Carries exact
* counts and names the targets that hold replies, so the nudge is actionable rather than a
* content-free "keep working" that would cost the lead a turn just to discover what there is to
* do (constraint 5).
*
* @param pendingReplies total undrained worker replies across owned targets
* @param doneSessions session count in the DONE state — finished turns awaiting teardown
* @param liveWorkers sessions still registered (acquired minus released)
* @param replyTargets the terminals whose inboxes currently hold at least one reply
*/
record FleetState(int pendingReplies, int doneSessions, int liveWorkers, List<String> replyTargets) {
/** True when the lead has something real to collect — a reply or a session awaiting teardown. */
boolean hasPending() {
return pendingReplies > 0 || doneSessions > 0;
}
/** The nudge body, phrased for the two cases the heartbeat distinguishes. */
String nudgeText() {
StringBuilder sb = new StringBuilder(
"Heartbeat: you are idle and no bridge_send is waiting on you.");
if (hasPending()) {
sb.append(" The fleet has state to collect: ").append(pendingDetail());
} else {
sb.append(" Fleet is quiet — nothing pending to collect (")
.append(pendingReplies).append(" worker replies, ")
.append(doneSessions).append(" DONE sessions); ")
.append(liveWorkers).append(" workers live. You may stand down until new state re-arms me.");
}
return sb.toString();
}
private String pendingDetail() {
StringBuilder sb = new StringBuilder();
sb.append(pendingReplies).append(" worker ")
.append(pendingReplies == 1 ? "reply" : "replies")
.append(" pending collection");
if (!replyTargets.isEmpty()) {
// Render each as the exact command so the lead can act without parsing: the nearest
// analogue to ReplyPushLoop's bridge_poll(target=...) nudge.
sb.append(" (")
.append(String.join(", ",
replyTargets.stream().map(t -> "bridge_poll(target=" + t + ")").toList()))
.append(")");
}
sb.append(", ").append(doneSessions).append(" DONE session").append(doneSessions == 1 ? "" : "s")
.append(" awaiting teardown")
.append(", ").append(liveWorkers).append(" worker").append(liveWorkers == 1 ? "" : "s")
.append(" live");
return sb.toString();
}
}
/**
* Build a {@link FleetState} snapshot over the live roster and the reply inbox — the same
* sources the metrics collector uses, so the heartbeat reports facts the operator can cross-check
* against {@code /metrics}. Package-private and pure (reads only) so it is unit-testable.
*/
static FleetState snapshot(ReplyInbox inbox, Supplier<List<WorkerSession>> roster) {
List<WorkerSession> sessions = roster.get();
int replies = 0;
int done = 0;
List<String> replyTargets = new ArrayList<>();
for (WorkerSession s : sessions) {
if (s.terminalId() == null) {
continue;
}
int depth = inbox.peek(s.terminalId()).size();
if (depth > 0) {
replies += depth;
replyTargets.add(s.terminalId());
}
if (s.state() == WorkerSession.State.DONE) {
done++;
}
}
return new FleetState(replies, done, sessions.size(), List.copyOf(replyTargets));
}
}
@@ -175,6 +175,17 @@ public final class ReplyPushLoop {
// --- lifecycle -----------------------------------------------------------------------------
/**
* Whether any reminder loop is currently active for some target (CB-551). The idle-lead heartbeat
* uses this to stand aside: while the push loop is actively nudging the lead, a concurrent
* heartbeat injection would start a second competing turn in the same pane — racing loops multiply
* turns and context burn. "Active" means a schedule exists in {@link #activeTargets}; the set is
* bounded by what has been triggered, not by any persistent state.
*/
public boolean isActive() {
return !activeTargets.isEmpty();
}
/** Shut down the scheduler. Outstanding reminders are cancelled. */
public void stop() {
scheduler.shutdownNow();
@@ -132,6 +132,7 @@ class BridgedConfigTest {
leaders: {}
architects: {}
leadScan: {}
leadHeartbeat: {}
placement: fixed
auth: {}
""").isEmpty(), "the known-key set must not drift from the record components");
@@ -180,6 +181,51 @@ class BridgedConfigTest {
assertEquals(30, scan.intervalSeconds());
}
// ── CB-551: the idle-lead heartbeat ─────────────────────────────────────────────────────────
/**
* Acceptance (f): absent config ⇒ the heartbeat is OFF completely. It spends the operator's
* subscription on its own initiative, so an upgraded daemon must never switch it on unasked.
*/
@Test
void leadHeartbeatIsOffUnlessTheBlockIsPresent(@TempDir Path dir) throws Exception {
Path f = dir.resolve("no-heartbeat.yaml");
Files.writeString(f, "bind:\n port: 8080\n");
assertNull(BridgedConfig.load(f).leadHeartbeat(),
"no leadHeartbeat: block ⇒ the loop is never constructed and never fires (CB-551)");
}
@Test
void leadHeartbeatDefaultsItsFieldsWhenTheBlockIsPresentButBare(@TempDir Path dir) throws Exception {
Path f = dir.resolve("bare-heartbeat.yaml");
Files.writeString(f, "bind:\n port: 8080\nleadHeartbeat: {}\n");
BridgedConfig.LeadHeartbeat hb = BridgedConfig.load(f).leadHeartbeat();
assertNotNull(hb);
assertEquals(300, hb.idleAfterSeconds(), "default quiet period");
assertEquals(60_000L, hb.backoffMs(), "default backoff");
assertEquals(3, hb.quietNudgeCap(), "default quiet-nudge cap");
}
@Test
void leadHeartbeatReadsExplicitValues(@TempDir Path dir) throws Exception {
Path f = dir.resolve("heartbeat.yaml");
Files.writeString(f, """
bind:
port: 8080
leadHeartbeat:
idleAfterSeconds: 600
backoffMs: 45000
quietNudgeCap: 5
""");
BridgedConfig.LeadHeartbeat hb = BridgedConfig.load(f).leadHeartbeat();
assertEquals(600, hb.idleAfterSeconds());
assertEquals(45_000L, hb.backoffMs());
assertEquals(5, hb.quietNudgeCap());
}
/**
* The hazard the guard exists for: bridged writes worker tab labels and reads lead tab labels.
* Overlap the two and every worker it spawns is read back as a lead.
@@ -835,6 +881,10 @@ class BridgedConfigTest {
terminal: term_abc123
pushReminders: 5
pushBackoffMs: 15000
leadHeartbeat:
idleAfterSeconds: 600
backoffMs: 45000
quietNudgeCap: 5
""");
BridgedConfig cfg = BridgedConfig.load(f);
@@ -860,6 +910,9 @@ class BridgedConfigTest {
assertEquals("term_abc123", cfg.primary().terminal());
assertEquals(5, cfg.primary().remindersOrDefault());
assertEquals(15000L, cfg.primary().backoffMsOrDefault());
assertEquals(600, cfg.leadHeartbeat().idleAfterSeconds(), "leadHeartbeat binds at the top level");
assertEquals(45_000L, cfg.leadHeartbeat().backoffMs());
assertEquals(5, cfg.leadHeartbeat().quietNudgeCap());
}
@Test
@@ -0,0 +1,215 @@
package dev.ltms.bridged.msg;
import dev.ltms.bridged.herdr.AgentStatus;
import dev.ltms.bridged.mcp.PrimaryRegistry;
import dev.ltms.bridged.session.WorkerSession;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
/**
* Unit tests for the CB-551 idle-lead heartbeat: the pure {@link LeadHeartbeatLoop#decide} decision
* function, the nudge text, and the {@link LeadHeartbeatLoop#snapshot} fleet snapshot.
*
* <p>All decision tests call {@code decide} directly with explicit nanoTime values from an injected
* clock — no sleeping, no scheduler races. This mirrors how {@code ReplyPushLoopTest} pins the pure
* decision before exercising the loop.
*/
class LeadHeartbeatLoopTest {
/** Arbitrary nanoTime origin for the fake clock. */
private static final long NOW = 1_000_000_000L;
/** 300s (the default quiet period) in nanos — under it the lead is "within the quiet period". */
private static final long IDLE_AFTER_NANOS = TimeUnit.SECONDS.toNanos(300);
/** 400s of idle — clearly past the quiet period. */
private static final long IDLE_PAST = NOW - TimeUnit.SECONDS.toNanos(400);
/** 10s of idle — clearly within the quiet period. */
private static final long IDLE_WITHIN = NOW - TimeUnit.SECONDS.toNanos(10);
private ScheduledExecutorService scheduler;
@BeforeEach
void setUp() {
scheduler = Executors.newSingleThreadScheduledExecutor();
}
@AfterEach
void tearDown() {
scheduler.shutdownNow();
}
/** A fleet with nothing pending. */
private static LeadHeartbeatLoop.FleetState quietFleet() {
return new LeadHeartbeatLoop.FleetState(0, 0, 2, List.of());
}
/** A fleet with a pending reply and a DONE session. */
private static LeadHeartbeatLoop.FleetState pendingFleet() {
return new LeadHeartbeatLoop.FleetState(2, 1, 3, List.of("term_a", "term_b"));
}
/** The loop under test; the scheduler is never invoked on the pure decide path. */
private static LeadHeartbeatLoop loop(int quietCap) {
return new LeadHeartbeatLoop(
new PrimaryRegistry("term_lead"), null /*agents — unused on the decide path*/,
null /*inbox*/, List::of, null /*pushLoop*/, null /*scheduler*/, () -> 0L,
IDLE_AFTER_NANOS, 1_000L, quietCap);
}
// ── (a) a working lead is never injected ───────────────────────────────────────────────────
@Test
void aWorkingLeadIsNeverInjected() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 0, AgentStatus.WORKING, false, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.LEAD_BUSY, d.action(),
"a WORKING lead is making progress and must not be touched");
assertNull(d.idleSinceNanos(), "a busy lead resets the idle window");
assertEquals(0, d.quietCount(), "a busy lead re-arms the quiet counter");
}
@Test
void anUnreadableStatusIsNeverInjectedEither() {
// A failed status read (or a gone agent) must degrade to "do not inject", never hammer the pane.
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 0, AgentStatus.UNKNOWN, false, true, pendingFleet());
assertEquals(LeadHeartbeatLoop.Action.LEAD_BUSY, d.action(),
"never inject into a state the loop cannot read");
}
// ── (b) an idle lead within the quiet period is not yet injected ───────────────────────────
@Test
void justBecameIdleStartsTheDebounceWindow() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, null, 0, AgentStatus.IDLE, false, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.WAIT_IDLE, d.action(),
"the first injectable tick only records the start of the idle stretch");
assertEquals(NOW, d.idleSinceNanos(), "the idle window opens at the moment the lead became injectable");
}
@Test
void idleWithinQuietPeriodIsNotInjected() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_WITHIN, 0, AgentStatus.IDLE, false, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.WAIT_IDLE, d.action(),
"a lead idle for 10s (< 300s) has just finished a turn — do not re-prompt it");
}
// ── (c) an idle lead past the quiet period is injected ─────────────────────────────────────
@Test
void idlePastQuietPeriodIsInjected() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 0, AgentStatus.IDLE, false, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.INJECT, d.action(),
"a lead continuously idle past the quiet period is the reason to nudge");
}
@Test
void blockedAndDoneAreInjectableViewsOfIdle() {
assertEquals(LeadHeartbeatLoop.Action.INJECT, loop(3).decide(
NOW, IDLE_PAST, 0, AgentStatus.BLOCKED, false, true, quietFleet()).action());
assertEquals(LeadHeartbeatLoop.Action.INJECT, loop(3).decide(
NOW, IDLE_PAST, 0, AgentStatus.DONE, false, true, quietFleet()).action());
}
@Test
void nothingIsInjectedWhenNoLeadIsKnown() {
var d = loop(3).decide(NOW, IDLE_PAST, 0, AgentStatus.IDLE, false, false, pendingFleet());
assertEquals(LeadHeartbeatLoop.Action.WAIT_IDLE, d.action(),
"with no known lead there is nobody to nudge — keep waiting until one is discovered");
assertEquals(IDLE_PAST, d.idleSinceNanos(), "the idle window stays open so discovery re-arms it");
}
// ── (d) the quiet-nudge cap stops the loop ──────────────────────────────────────────────────
@Test
void quietNudgeCapStopsTheLoopWhenNothingIsPending() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 3, AgentStatus.IDLE, false, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.QUIET_DONE, d.action(),
"3 consecutive nothing-pending nudges have already happened — stop nagging an empty fleet");
}
// ── (e) new pending state resets the cap and re-arms the loop ───────────────────────────────
@Test
void newPendingStateResetsTheQuietCap() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 3, AgentStatus.IDLE, false, true, pendingFleet());
assertEquals(LeadHeartbeatLoop.Action.INJECT, d.action(),
"real state appearing re-arms the loop past an exhausted cap");
assertEquals(0, d.quietCount(), "the pending state resets the consecutive-quiet counter");
}
// ── constraint 6: never race ReplyPushLoop ─────────────────────────────────────────────────
@Test
void standsDownWhileReplyPushLoopIsActive() {
LeadHeartbeatLoop.Decision d = loop(3).decide(
NOW, IDLE_PAST, 2, AgentStatus.IDLE, true, true, quietFleet());
assertEquals(LeadHeartbeatLoop.Action.STAND_DOWN, d.action(),
"a second injection would start a competing turn — stand aside instead");
assertEquals(0, d.quietCount(), "the active push is real state, so it re-arms the cap");
}
// ── nudge text (constraint 5: the nudge must carry state) ──────────────────────────────────
@Test
void pendingNudgeCarriesTheFleetFacts() {
String t = pendingFleet().nudgeText();
assertTrue(t.startsWith("Heartbeat:"), "the nudge identifies itself as a heartbeat");
assertTrue(t.contains("2 worker replies pending collection"), t);
assertTrue(t.contains("bridge_poll(target=term_a)"), t);
assertTrue(t.contains("bridge_poll(target=term_b)"), t);
assertTrue(t.contains("1 DONE session awaiting teardown"), t);
assertTrue(t.contains("3 workers live"), t);
}
@Test
void nothingPendingNudgeSaysExactlyThatSoTheLeadCanStandDown() {
String t = quietFleet().nudgeText();
assertTrue(t.contains("nothing pending to collect"), t);
assertTrue(t.contains("0 worker replies"), t);
assertTrue(t.contains("0 DONE sessions"), t);
assertTrue(t.contains("2 workers live"), t);
assertTrue(t.contains("stand down"), "allow the lead to stand down rather than hunt");
}
// ── snapshot over roster + inbox ───────────────────────────────────────────────────────────
@Test
void snapshotCountsRepliesDoneSessionsAndLiveWorkers() {
InMemoryReplyInbox inbox = new InMemoryReplyInbox();
inbox.own("term_w1");
inbox.publish("term_w1", "m1", "hello");
WorkerSession done = new WorkerSession("p1", "term_w1", "prof", "/cwd", null,
0, 0, 0, WorkerSession.State.DONE, null, null);
WorkerSession ready = new WorkerSession("p2", "term_w2", "prof", "/cwd", null,
0, 0, 0, WorkerSession.State.READY, null, null);
LeadHeartbeatLoop.FleetState fs = LeadHeartbeatLoop.snapshot(inbox, () -> List.of(done, ready));
assertEquals(1, fs.pendingReplies(), "one undrained reply on term_w1");
assertEquals(1, fs.doneSessions(), "term_w1 is DONE, awaiting teardown");
assertEquals(2, fs.liveWorkers(), "both sessions are still registered");
assertEquals(List.of("term_w1"), fs.replyTargets(), "the target with a pending reply is named");
assertTrue(fs.hasPending(), "a pending reply counts as fleet state to collect");
}
@Test
void snapshotWithEmptyRosterHasNoPendingState() {
InMemoryReplyInbox inbox = new InMemoryReplyInbox();
LeadHeartbeatLoop.FleetState fs = LeadHeartbeatLoop.snapshot(inbox, List::of);
assertFalse(fs.hasPending());
assertEquals(0, fs.liveWorkers());
}
}
@@ -179,6 +179,20 @@ class ReplyPushLoopTest {
// --- nudge format --------------------------------------------------------------------------
@Test
void isActiveReflectsALiveScheduleForTheHeartbeatStandDown() {
var rec = recordingClient();
agents = new AgentControl(rec);
inbox.publish(WORKER, "m1", "hello");
ReplyPushLoop loop = loop(1, 100_000); // long backoff so the tick cannot fire mid-test
loop.onReplyQueued(WORKER);
assertTrue(loop.isActive(), "CB-551: the heartbeat must stand aside while a reminder is live");
loop.stop();
assertFalse(loop.isActive(), "stopping clears the active schedule");
}
@Test
void nudgeFormatIsCorrect() {
String nudge = ReplyPushLoop.NUDGE_FORMAT.formatted(WORKER, WORKER);