broker: uriEnv config + non-fatal unreachable broker on boot
CI / build (pull_request) Failing after 56s
CI / contract (pull_request) Successful in 1m6s

#151: Broker gains uriEnv beside uri, taking the AMQP URI from an env var so
the password stays out of fleetd.yaml (same pattern as auth.tokenEnv). uriEnv
wins when set; isConfigured() treats a uriEnv naming an unset/blank variable
as unconfigured. A configured uriEnv is added to the startup required-secrets
report. Never logs the resolved URI (it carries the password).

#152: AmqpReplyInbox.open throwing at boot no longer stops the daemon. The
selection at the call site catches the failure and falls back to the in-memory
inbox for the process lifetime, warning loudly that durable cross-restart
delivery is off and logging the failed URI with credentials stripped.
This commit is contained in:
Dai Ha
2026-08-23 13:49:26 +02:00
parent 83b50753fe
commit 95a8dbcea9
4 changed files with 335 additions and 19 deletions
@@ -371,18 +371,10 @@ public final class Fleetd {
StatusPoller poller = new StatusPoller(agents, injector, Injector.POLL_INTERVAL_MILLIS);
poller.start();
// CB-307: reply inbox. A broker: block (with a uri) selects the AMQP-backed durable adapter;
// absent, bridged stays soft-state on the in-memory inbox. The AMQP inbox owns a broker
// CB-307: reply inbox. A broker: block selects the AMQP-backed durable adapter; absent (or
// unusable), bridged stays soft-state on the in-memory inbox. The AMQP inbox owns a broker
// connection, so keep the reference to close it in the ordered shutdown hook.
final ReplyInbox replyInbox;
if (cfg.broker() != null && cfg.broker().isConfigured()) {
replyInbox = AmqpReplyInbox.open(cfg.broker().uri(), cfg.broker().prefetchOrDefault());
log.info("reply inbox: AMQP broker (durable) at {} (prefetch={})",
cfg.broker().uri(), cfg.broker().prefetchOrDefault());
} else {
replyInbox = new InMemoryReplyInbox();
log.info("reply inbox: in-memory (soft-state)");
}
final ReplyInbox replyInbox = selectReplyInbox(cfg.broker(), System.getenv(), AmqpReplyInbox::open);
// CB-307: learn the primary's terminal from orchestration tool calls (or pin from config).
// The pin also feeds CallerResolver below: a primary running inside a herdr pane would
// otherwise resolve as a worker and be refused every orchestration tool.
@@ -577,14 +569,101 @@ public final class Fleetd {
return target -> presence.isPresent(target) || leads.get().containsKey(target);
}
/** Injection seam for {@link #selectReplyInbox}: production binds {@link AmqpReplyInbox#open}. */
@FunctionalInterface
interface AmqpOpener {
ReplyInbox open(String uri, int prefetch);
}
/**
* CB-151/152: pick the reply inbox. A usable broker — a literal {@code uri}, or a {@code
* uriEnv} whose variable resolves (both read from {@code env}) — selects the durable AMQP inbox.
* Everything else falls back to the in-memory inbox: no broker block, a blank {@code uri}, a
* {@code uriEnv} whose variable is unset or blank, or a broker unreachable at boot. The two
* lossy paths warn <em>loudly</em> — never silently — because what is lost is durable,
* cross-restart reply delivery. Package-private and env-injected so the selection is testable
* without a real broker or a mutable process environment.
*/
static ReplyInbox selectReplyInbox(FleetConfig.Broker broker, Map<String, String> env, AmqpOpener amqp) {
if (broker == null) {
log.info("reply inbox: in-memory (soft-state)");
return new InMemoryReplyInbox();
}
if (broker.hasUriEnv()) {
// uriEnv is authoritative whenever set (CB-151): the operator moved off clear text, so
// it must not quietly fall back onto a stale literal uri.
if (broker.uri() != null && !broker.uri().isBlank()) {
log.info("broker.uri is ignored because broker.uriEnv={} is set", broker.uriEnv());
}
String effectiveUri = broker.effectiveUri(env);
if (effectiveUri == null) {
log.warn("broker.uriEnv={} is unset or blank — durable AMQP reply inbox DISABLED. "
+ "Replies are soft-state and will not survive a restart. Set {} in the "
+ "daemon's environment (see scripts/redeploy-bridged.sh) and restart to "
+ "use the durable broker inbox.",
broker.uriEnv(), broker.uriEnv());
log.info("reply inbox: in-memory (soft-state)");
return new InMemoryReplyInbox();
}
log.info("reply inbox: AMQP broker (durable) via env var {} (prefetch={})",
broker.uriEnv(), broker.prefetchOrDefault());
return openAmqpOrFallback(effectiveUri, broker.prefetchOrDefault(), "uriEnv " + broker.uriEnv(), amqp);
}
// No uriEnv: the literal uri path (existing behaviour).
if (broker.effectiveUri(env) == null) {
log.info("reply inbox: in-memory (soft-state)");
return new InMemoryReplyInbox();
}
log.info("reply inbox: AMQP broker (durable) (prefetch={})", broker.prefetchOrDefault());
return openAmqpOrFallback(broker.uri(), broker.prefetchOrDefault(), "uri", amqp);
}
/**
* Open the AMQP inbox, falling back to the in-memory inbox for this process lifetime if the
* broker cannot be reached at boot (CB-152). Not silent: the warning says durable delivery is
* off, replies are soft-state and will not survive a restart, plus the source that failed and
* the URI <em>with credentials stripped</em>. Never retries in the background — a broker that
* drops <em>after</em> startup already self-heals via the connection factory's automatic
* recovery; only the boot path is changed here.
*/
private static ReplyInbox openAmqpOrFallback(String effectiveUri, int prefetch, String source,
AmqpOpener amqp) {
try {
return amqp.open(effectiveUri, prefetch);
} catch (IllegalStateException e) {
log.warn("cannot reach AMQP broker ({}, {}) — falling back to the in-memory reply inbox "
+ "for this process lifetime. Durable, cross-restart reply delivery is OFF; "
+ "replies are soft-state and will not survive a restart. Reason: {}",
source, stripCredentials(effectiveUri), reasonOf(e));
log.info("reply inbox: in-memory (soft-state)");
return new InMemoryReplyInbox();
}
}
/** An AMQP URI carries {@code user:pass@} inline — show the host/port, never the credentials. */
static String stripCredentials(String uri) {
return uri == null ? null : uri.replaceAll("://[^@/]*@", "://");
}
/** The deepest cause's class and message — the outermost {@code IllegalStateException} echoes the URI (with password). */
private static String reasonOf(Throwable e) {
Throwable t = e;
while (t.getCause() != null && t.getCause() != t) {
t = t.getCause();
}
String msg = t.getMessage();
return t.getClass().getSimpleName() + (msg == null || msg.isBlank() ? "" : ": " + msg);
}
/**
* CB-594: which env vars the loaded config actually needs, and why — every non-{@code
* subscription} profile's {@code tokenEnv} (a subscription profile never reads one, see
* {@link FleetConfig.Profile#isSubscription()}), plus every profile's {@code gitTokenEnv}
* where set (opt-in). Derived from the config, not hard-coded, so a new profile is covered for
* free. A var required by more than one profile is one entry naming every profile that needs
* it. Deliberately excludes {@code auth.tokenEnv}: that one is already enforced loudly, by a
* startup throw in {@code main()} — about 370 lines <em>below</em> this method's call site
* where set (opt-in), plus a configured {@code broker.uriEnv} (CB-151). Derived from the
* config, not hard-coded, so a new profile is covered for free. A var required by more than one
* profile is one entry naming every profile that needs it. Deliberately excludes {@code
* auth.tokenEnv}: that one is already enforced loudly, by a startup throw in {@code main()} —
* about 370 lines <em>below</em> this method's call site
* ({@link #reportRequiredSecrets(FleetConfig)}), not a few lines above it. That throw only
* fires when {@code auth.mode: token} is configured; under the default loopback-trust mode it
* never runs, and {@code auth.tokenEnv} is simply not required.
@@ -604,6 +683,11 @@ public final class Fleetd {
.add("profile '" + name + "' gitTokenEnv");
}
});
FleetConfig.Broker broker = cfg.broker();
if (broker != null && broker.hasUriEnv()) {
requiredBy.computeIfAbsent(broker.uriEnv(), _ -> new ArrayList<>())
.add("broker uriEnv");
}
return requiredBy;
}
@@ -551,16 +551,52 @@ public record FleetConfig(
*
* @param uri AMQP connection URI, e.g. {@code amqp://guest:guest@127.0.0.1:5672/}. Blank/
* {@code null} ⇒ the broker block is treated as absent (in-memory adapter).
* Ignored when {@code uriEnv} is set.
* @param uriEnv name of a host env var holding the AMQP URI (CB-151). The URI carries its
* credentials inline, so giving the variable <em>name</em> keeps the password
* out of the config file, same as {@code auth.tokenEnv}/{@code Profile.tokenEnv}.
* Wins over {@code uri} whenever set. Blank/{@code null} ⇒ ignored.
* @param prefetch CB-527: the consumer's {@code basicQos} prefetch count, bounding how many
* unacked messages the AMQP inbox holds in-heap per owned target. {@code null}/
* non-positive ⇒ {@link AmqpReplyInbox#DEFAULT_PREFETCH}.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public record Broker(String uri, Integer prefetch) {
public record Broker(String uri, String uriEnv, Integer prefetch) {
/** True when a usable broker URI is configured (an empty block does not enable AMQP). */
/** True when a {@code uriEnv} is configured by name, whether or not its variable resolves. */
public boolean hasUriEnv() {
return uriEnv != null && !uriEnv.isBlank();
}
/**
* True when a usable broker URI is configured (an empty block does not enable AMQP).
* Honors {@code uriEnv} first: if it names a variable that is unset or blank, the broker is
* <em>not</em> configured (the daemon falls back to the in-memory inbox) — a bare {@code uri}
* is only consulted when no {@code uriEnv} is set. Env lookup makes this process-dependent;
* callers already reading {@link System#getenv} are the right ones to invoke it.
*/
public boolean isConfigured() {
return uri != null && !uri.isBlank();
return effectiveUri() != null;
}
/**
* The effective AMQP URI to connect with. {@code uriEnv} wins when set (both over {@code uri}
* and alone). When {@code uriEnv} names a variable that is unset or blank, returns {@code
* null} rather than falling back to {@code uri} — an operator who moved to the secret store
* must not silently drop back onto a stale clear-text URI. Returns the literal {@code uri}
* when no {@code uriEnv} is configured.
*/
public String effectiveUri() {
return effectiveUri(System.getenv());
}
/** As {@link #effectiveUri()}, reading the variable value from {@code env} (the injection seam). */
public String effectiveUri(Map<String, String> env) {
if (hasUriEnv()) {
String value = env.get(uriEnv);
return (value != null && !value.isBlank()) ? value : null;
}
return (uri != null && !uri.isBlank()) ? uri : null;
}
/** The prefetch to use, defaulting to {@link AmqpReplyInbox#DEFAULT_PREFETCH} when unset. */