Dai Ha
e501d39988
A BACKEND_EXHAUSTED classification (stage A) now puts that profile's credential into a BackendQuarantine for a configurable cooldown. A spawn onto a quarantined profile is refused naming the credential and roughly when it lifts; weighted/round-robin/fixed placement skip a quarantined candidate; the quarantine lifts itself on the injected clock; and it is visible on bridge_profiles. Keyed by credential, not by profile name, via the new Profile.credentialId (profiles sharing one credential quarantine together — e.g. two models on one account) and effectiveCredentialId() (unset ⇒ quarantines alone, today's behaviour unchanged). Fixed a related gap along the way: a reload changing exhaustedPattern was silently reported "applied" even though it's deferred — sameLaunchSettings() now catches it too.
292 lines
14 KiB
Java
292 lines
14 KiB
Java
package dev.ltms.bridged.config;
|
|
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import java.nio.file.Path;
|
|
import java.util.ArrayList;
|
|
import java.util.LinkedHashSet;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Objects;
|
|
import java.util.Set;
|
|
import java.util.concurrent.atomic.AtomicReference;
|
|
import java.util.function.Supplier;
|
|
|
|
/**
|
|
* The daemon's live configuration, re-readable without a restart (CB-559).
|
|
*
|
|
* <p>Consumers hold this, not a {@link BridgedConfig}, and read through {@link #get()} at the point
|
|
* of use. A component that captures {@code ref.get()} into a field at construction has opted out of
|
|
* reload — which is sometimes right (see <em>deferred</em> below), but it must then be a deliberate
|
|
* choice rather than an accident of where the field was initialised.
|
|
*
|
|
* <h2>Not every key can change under a running daemon</h2>
|
|
* Keys fall into three classes, and the difference is about what already exists when the reload
|
|
* happens — not about how important the key is.
|
|
*
|
|
* <ul>
|
|
* <li><strong>Hot</strong> — re-read per use, so a reload takes effect on the next spawn:
|
|
* {@code fleet:} (every role pool, {@code charters}, and {@code tabLabel}),
|
|
* {@code placement:}, and an existing profile's {@code weight} / {@code maxLoad}. Those
|
|
* three are read through a supplier on {@code CompositePeerLauncher}, which is what makes
|
|
* them hot — not the fact that they are config. <strong>This does NOT include
|
|
* {@code fleet.leaders}</strong>: {@code Bridged.main} reads {@code cfg.fleet().leaders()}
|
|
* once at startup to build the {@code LeadTabScanner} and the {@code LeadLauncher}, and
|
|
* neither is reconstructed on reload — so a lead added, removed, or re-{@code tab}'d under
|
|
* {@code fleet.leaders} needs a restart, the same as any deferred key below.</li>
|
|
* <li><strong>Deferred</strong> — accepted into the new snapshot, but the wiring built at startup
|
|
* keeps the old value until a restart: {@code lifecycle:}, {@code leadHeartbeat:},
|
|
* {@code spawnReadyTimeoutMs} / {@code spawnReadyPollMs}, {@code quarantineCooldownSeconds}
|
|
* (CB-578 stage B — baked once into the {@code BackendQuarantine} built at startup),
|
|
* {@code guard:}, {@code worktreeRoot:}, adding or removing a profile (a new backend needs its own launcher,
|
|
* which is constructed once), <em>and an existing profile's launch settings</em> —
|
|
* {@code model}, {@code baseUrl}, {@code argv}, {@code env}, {@code mcpUrl},
|
|
* {@code exhaustedPattern} (CB-578 stage A — compiled once into {@code Bridged.main}'s
|
|
* pattern map at startup), and the rest. {@code credentialId} (CB-578 stage B) is NOT on
|
|
* this list — it is read live off the config supplier at every quarantine check and
|
|
* exhaustion event, exactly like {@code weight} / {@code maxLoad}, so it is hot instead.
|
|
* {@code HerdrPeerLauncher} takes {@code Map.copyOf(profiles)} at construction and resolves
|
|
* each spawn out of that copy, so those never reach a launch until the daemon restarts. A
|
|
* reload logs these rather than pretending they applied.</li>
|
|
* <li><strong>Cold</strong> — cannot change at all under a running daemon: {@code bind:},
|
|
* {@code herdrSocket:}, {@code broker:} and {@code auth:}. The socket is bound, the broker
|
|
* connection is open, and the auth mode decides who may reach the port that is already
|
|
* listening.</li>
|
|
* </ul>
|
|
*
|
|
* <p><strong>A cold change refuses the whole reload.</strong> Not the hot half applied and the cold
|
|
* half warned about: that would leave the running daemon in a state matching no file on disk, which
|
|
* is the worst thing a reload can do to an operator debugging one. Refusing keeps the invariant that
|
|
* the live config is always some version of the file, and the message names the keys that must
|
|
* change through a restart.
|
|
*
|
|
* <p>A reload that fails to parse or fails validation is also refused, and the previous config keeps
|
|
* running. A config file being edited is normally read once mid-save; degrading a working daemon
|
|
* because it caught a half-written file would be a bad trade.
|
|
*/
|
|
public final class ConfigRef implements Supplier<BridgedConfig> {
|
|
|
|
private static final Logger log = LoggerFactory.getLogger(ConfigRef.class);
|
|
|
|
/** Keys that cannot change under a running daemon — see the class doc. */
|
|
private static final Set<String> COLD_KEYS =
|
|
Set.of("bind", "herdrSocket", "broker", "auth");
|
|
|
|
private final Path path;
|
|
private final AtomicReference<BridgedConfig> current;
|
|
|
|
public ConfigRef(Path path, BridgedConfig initial) {
|
|
this.path = path;
|
|
this.current = new AtomicReference<>(Objects.requireNonNull(initial, "initial config"));
|
|
}
|
|
|
|
/** A fixed reference that never reloads — for tests and for wiring built from a config in code. */
|
|
public static ConfigRef fixed(BridgedConfig cfg) {
|
|
return new ConfigRef(null, cfg);
|
|
}
|
|
|
|
/** The live configuration. Read this per use; do not cache it in a field. */
|
|
@Override
|
|
public BridgedConfig get() {
|
|
return current.get();
|
|
}
|
|
|
|
/** The file this ref reloads from, or {@code null} for a {@link #fixed} ref. */
|
|
public Path path() {
|
|
return path;
|
|
}
|
|
|
|
/**
|
|
* What a reload attempt did.
|
|
*
|
|
* @param applied true when the new config is now live
|
|
* @param coldKeys cold keys whose value changed, which is why an unapplied reload was refused
|
|
* @param deferred keys that changed and were accepted, but whose effect waits for a restart
|
|
* @param error the parse or validation failure that refused the reload, else {@code null}
|
|
*/
|
|
public record Outcome(boolean applied, List<String> coldKeys, List<String> deferred,
|
|
String error) {
|
|
|
|
public Outcome {
|
|
coldKeys = List.copyOf(coldKeys);
|
|
deferred = List.copyOf(deferred);
|
|
}
|
|
|
|
static Outcome refusedCold(List<String> keys) {
|
|
return new Outcome(false, keys, List.of(), null);
|
|
}
|
|
|
|
static Outcome failed(String error) {
|
|
return new Outcome(false, List.of(), List.of(), error);
|
|
}
|
|
|
|
/** A one-line summary for the operator — the reason, not just the verdict. */
|
|
public String summary() {
|
|
if (error != null) {
|
|
return "config reload refused — " + error;
|
|
}
|
|
if (!applied) {
|
|
return "config reload refused — these keys cannot change under a running daemon: "
|
|
+ String.join(", ", coldKeys) + ". Restart bridged to apply them.";
|
|
}
|
|
if (!deferred.isEmpty()) {
|
|
return "config reloaded; these changes need a restart to take effect: "
|
|
+ String.join(", ", deferred);
|
|
}
|
|
return "config reloaded";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Re-read the file, validate it, and swap it in when nothing cold changed.
|
|
*
|
|
* <p>Never throws: a reload is a best-effort operation on a daemon that is already serving, and
|
|
* a bad edit must not take it down. Every failure path leaves the previous config live and is
|
|
* reported through the returned {@link Outcome}.
|
|
*/
|
|
public Outcome reload() {
|
|
if (path == null) {
|
|
return Outcome.failed("this config was built in code and has no file to reload from");
|
|
}
|
|
BridgedConfig old = current.get();
|
|
BridgedConfig fresh;
|
|
try {
|
|
fresh = BridgedConfig.load(path);
|
|
// The same gate startup runs. A config that would have refused to boot must not be able
|
|
// to slip in through a reload — that is how a daemon ends up in a state it could never
|
|
// have started in, which is the hardest kind to debug.
|
|
fresh.validateAuthExposure();
|
|
fresh.validateLeadTabPrefixes();
|
|
fresh.validateSubscriptionProfiles();
|
|
fresh.validateCharters();
|
|
fresh.validateMembers();
|
|
} catch (RuntimeException e) {
|
|
String msg = e.getMessage() == null ? e.toString() : e.getMessage();
|
|
log.warn("config reload from {} refused, keeping the running config: {}", path, msg);
|
|
return Outcome.failed(msg);
|
|
}
|
|
|
|
List<String> cold = changedColdKeys(old, fresh);
|
|
if (!cold.isEmpty()) {
|
|
Outcome out = Outcome.refusedCold(cold);
|
|
log.warn(out.summary());
|
|
return out;
|
|
}
|
|
|
|
List<String> deferred = changedDeferredKeys(old, fresh);
|
|
current.set(fresh);
|
|
Outcome out = new Outcome(true, List.of(), deferred, null);
|
|
log.info(out.summary());
|
|
return out;
|
|
}
|
|
|
|
/** Cold keys whose value differs between the running config and the candidate. */
|
|
private static List<String> changedColdKeys(BridgedConfig old, BridgedConfig fresh) {
|
|
List<String> changed = new ArrayList<>();
|
|
if (!Objects.equals(old.bind(), fresh.bind())) {
|
|
changed.add("bind");
|
|
}
|
|
if (!Objects.equals(old.herdrSocket(), fresh.herdrSocket())) {
|
|
changed.add("herdrSocket");
|
|
}
|
|
if (!Objects.equals(old.broker(), fresh.broker())) {
|
|
changed.add("broker");
|
|
}
|
|
if (!Objects.equals(old.auth(), fresh.auth())) {
|
|
changed.add("auth");
|
|
}
|
|
// Kept in step with COLD_KEYS so the doc and the code cannot drift apart silently.
|
|
assert COLD_KEYS.containsAll(changed) : "a cold key was reported that COLD_KEYS omits";
|
|
return changed;
|
|
}
|
|
|
|
/** Changed keys that were accepted but whose effect waits for a restart. */
|
|
private static List<String> changedDeferredKeys(BridgedConfig old, BridgedConfig fresh) {
|
|
List<String> changed = new ArrayList<>();
|
|
if (!Objects.equals(old.lifecycle(), fresh.lifecycle())) {
|
|
changed.add("lifecycle");
|
|
}
|
|
if (!Objects.equals(old.leadHeartbeat(), fresh.leadHeartbeat())) {
|
|
changed.add("leadHeartbeat");
|
|
}
|
|
if (!Objects.equals(old.guard(), fresh.guard())) {
|
|
changed.add("guard");
|
|
}
|
|
if (!Objects.equals(old.worktreeRoot(), fresh.worktreeRoot())) {
|
|
changed.add("worktreeRoot");
|
|
}
|
|
if (!Objects.equals(old.spawnReadyTimeoutMs(), fresh.spawnReadyTimeoutMs())
|
|
|| !Objects.equals(old.spawnReadyPollMs(), fresh.spawnReadyPollMs())) {
|
|
changed.add("spawnReady*");
|
|
}
|
|
// CB-578 stage B: baked once into the BackendQuarantine built at startup — a running
|
|
// quarantine keeps its original cooldown regardless, and a new cooldown only applies to a
|
|
// quarantine that starts after a restart.
|
|
if (!Objects.equals(old.quarantineCooldownSeconds(), fresh.quarantineCooldownSeconds())) {
|
|
changed.add("quarantineCooldownSeconds");
|
|
}
|
|
Map<String, BridgedConfig.Profile> before =
|
|
old.profiles() == null ? Map.of() : old.profiles();
|
|
Map<String, BridgedConfig.Profile> after =
|
|
fresh.profiles() == null ? Map.of() : fresh.profiles();
|
|
// Adding or removing a profile is deferred: a new backend needs its own launcher, and
|
|
// launchers are built once at startup.
|
|
if (!before.keySet().equals(after.keySet())) {
|
|
Set<String> diff = new LinkedHashSet<>(before.keySet());
|
|
diff.addAll(after.keySet());
|
|
diff.removeIf(p -> before.containsKey(p) && after.containsKey(p));
|
|
changed.add("profiles (added/removed: " + String.join(", ", diff) + ")");
|
|
}
|
|
// An EXISTING profile's launch settings are deferred too, and this is easy to get wrong:
|
|
// `HerdrPeerLauncher` takes `Map.copyOf(profiles)` at construction and `spawn` resolves the
|
|
// profile out of that snapshot, so a reloaded model/baseUrl/argv/env never reaches a launch.
|
|
// Only weight and maxLoad are genuinely hot, because placement reads them through the
|
|
// supplier on the composite rather than from the adapter's copy. Without this check a
|
|
// changed model would report "config reloaded" and silently do nothing — the worst outcome
|
|
// a reload can produce, because the operator has no reason to doubt it.
|
|
List<String> relaunch = new ArrayList<>();
|
|
before.forEach((name, was) -> {
|
|
BridgedConfig.Profile now = after.get(name);
|
|
if (now != null && !sameLaunchSettings(was, now)) {
|
|
relaunch.add(name);
|
|
}
|
|
});
|
|
if (!relaunch.isEmpty()) {
|
|
changed.add("profiles." + String.join("/", relaunch) + " launch settings "
|
|
+ "(model, baseUrl, argv, env, …) — the launcher holds a startup snapshot");
|
|
}
|
|
return changed;
|
|
}
|
|
|
|
/**
|
|
* Whether two versions of a profile would launch a peer identically. Compares every component
|
|
* the launcher reads at spawn; {@code weight}, {@code maxLoad} and {@code credentialId} are
|
|
* excluded because those are read live (by the placement policy and, for credentialId, by
|
|
* {@code CompositePeerLauncher}/the CB-578 stage B exhaustion sink) and really do take effect on
|
|
* the next spawn.
|
|
*/
|
|
private static boolean sameLaunchSettings(BridgedConfig.Profile a, BridgedConfig.Profile b) {
|
|
return Objects.equals(a.baseUrl(), b.baseUrl())
|
|
&& Objects.equals(a.model(), b.model())
|
|
&& Objects.equals(a.configDir(), b.configDir())
|
|
&& Objects.equals(a.tokenEnv(), b.tokenEnv())
|
|
&& Objects.equals(a.argv(), b.argv())
|
|
&& Objects.equals(a.placement(), b.placement())
|
|
&& Objects.equals(a.workspace(), b.workspace())
|
|
&& Objects.equals(a.tabLabel(), b.tabLabel())
|
|
&& Objects.equals(a.mcpUrl(), b.mcpUrl())
|
|
&& Objects.equals(a.cwd(), b.cwd())
|
|
&& Objects.equals(a.parityOverlay(), b.parityOverlay())
|
|
&& Objects.equals(a.gitTokenEnv(), b.gitTokenEnv())
|
|
&& Objects.equals(a.gitHostEnv(), b.gitHostEnv())
|
|
&& Objects.equals(a.kind(), b.kind())
|
|
&& Objects.equals(a.env(), b.env())
|
|
&& Objects.equals(a.subscription(), b.subscription())
|
|
// CB-578 stage B: exhaustedPattern is compiled once into Bridged.main's pattern map
|
|
// at startup (see ExhaustedPatternLookup wiring) — a reload never re-reads it, so a
|
|
// changed pattern must be reported as deferred, exactly like model/baseUrl/argv.
|
|
&& Objects.equals(a.exhaustedPattern(), b.exhaustedPattern());
|
|
}
|
|
}
|