scripts/redeploy-fleetd.sh

@@ -96,13 +96,23 @@ SYSTEMD_UNIT='fleetd'
 # any of the prose detail strings and needs no escaping in a `case`/glob pattern.
 SUPERVISOR_DETAIL_SEP=$'\x1f'
 
-# fleetd #492 follow-up: set by systemd_loaded/systemd_installed when the underlying `systemctl`
-# call could not answer cleanly — it exited non-zero AND wrote something to stderr, which is a real
-# tool failure (e.g. it cannot reach the user bus over a non-lingering ssh session), never the same
-# fact as a clean negative answer ("not active", no stderr). Initialized here, not just inside the
-# probes, so detect_supervisor can read them under `set -u` even before either probe has ever run,
-# and so a test that stubs a probe with a plain `return 0`/`return 1` body (leaving these untouched)
-# reads a deterministic 0 rather than whatever a previous probe call left behind.
+# fleetd #492 follow-up, refined by fleetd #545: three states, not two, set by
+# systemd_loaded/systemd_installed —
+#   0 = no error, the probe ran and gave a clean answer.
+#   1 = the probe RAN and answered badly: `systemctl` exited non-zero AND wrote something to
+#       stderr, a real tool failure (e.g. it cannot reach the user bus over a non-lingering ssh
+#       session), never the same fact as a clean negative answer ("not active", no stderr).
+#   2 = the probe could not even be SET UP: the `mktemp` call that makes a place to capture
+#       `systemctl`'s stderr failed before `systemctl` ever ran. This is a fleetd #545 fix: on GNU
+#       coreutils (every Linux distribution) a template with no `X`s made `mktemp` fail every
+#       single time, and the two states were folded into one flag and one message that named
+#       cause 1 ("systemctl exited non-zero and reported an error on stderr") for a failure that
+#       was actually cause 2 — systemctl was never executed at all. One flag with two meanings
+#       needing different messages was the defect; a third value is the fix, not a second flag.
+# Initialized here, not just inside the probes, so detect_supervisor can read them under `set -u`
+# even before either probe has ever run, and so a test that stubs a probe with a plain
+# `return 0`/`return 1` body (leaving these untouched) reads a deterministic 0 rather than whatever
+# a previous probe call left behind.
 SYSTEMD_LOADED_ERRORED=0
 SYSTEMD_INSTALLED_ERRORED=0
 # fleetd #492 follow-up: SUPERVISOR_UNCLEAR_DETAIL is the specific supervisor/reason that
@@ -238,12 +248,17 @@ launchd_loaded() { launchctl list "$LAUNCHD_LABEL" >/dev/null 2>&1; }
 # fleetd #492 follow-up: both functions used to throw `systemctl`'s stderr straight into
 # /dev/null, which meant "systemctl answered no" and "systemctl could not answer at all" (e.g. it
 # cannot reach the user bus over a non-lingering ssh session) looked identical — both a plain
-# nonzero exit. They now capture stderr separately and set their own *_ERRORED flag ONLY when the
+# nonzero exit. They now capture stderr separately and set their own *_ERRORED flag to 1 when the
 # call exited non-zero AND wrote something to stderr — a real tool failure, never a clean "not
 # installed"/"not active" answer (which exits non-zero with empty stderr). detect_supervisor reads
 # the flag right after calling the probe, so a probe that could not answer routes to "unclear",
 # never silently becomes "none".
 #
+# fleetd #545: the flag has a third value, 2, set when the `mktemp` call that sets up the probe's
+# own stderr capture fails, before `systemctl` ever runs — see the SYSTEMD_LOADED_ERRORED /
+# SYSTEMD_INSTALLED_ERRORED comment above their initialization for why this is a third value on the
+# same flag, not a second flag.
+#
 # "installed": a unit FILE by this name exists, regardless of its current state — the systemd
 # analogue of the plist file existing on disk. `list-unit-files` reads unit definitions without
 # depending on runtime state, so this stays read-only and safe under --check.
@@ -251,8 +266,8 @@ systemd_installed() {
   SYSTEMD_INSTALLED_ERRORED=0
   command -v systemctl >/dev/null 2>&1 || return 1
   local err_file out rc=0
-  if ! err_file="$(mktemp -t systemd-installed-err)"; then
-    SYSTEMD_INSTALLED_ERRORED=1
+  if ! err_file="$(mktemp -t systemd-installed-err.XXXXXX)"; then
+    SYSTEMD_INSTALLED_ERRORED=2
     return 1
   fi
   out="$(systemctl --user list-unit-files "$SYSTEMD_UNIT.service" --no-legend 2>"$err_file")" || rc=$?
@@ -275,8 +290,8 @@ systemd_loaded() {
   SYSTEMD_LOADED_ERRORED=0
   command -v systemctl >/dev/null 2>&1 || return 1
   local err_file rc=0
-  if ! err_file="$(mktemp -t systemd-loaded-err)"; then
-    SYSTEMD_LOADED_ERRORED=1
+  if ! err_file="$(mktemp -t systemd-loaded-err.XXXXXX)"; then
+    SYSTEMD_LOADED_ERRORED=2
     return 1
   fi
   systemctl --user is-active "$SYSTEMD_UNIT" >/dev/null 2>"$err_file" || rc=$?
@@ -302,7 +317,7 @@ systemd_loaded() {
 # lost value instead of surfacing it (fleetd #497's defect class).
 unload_launchd_if_loaded() {
   local err_file rc=0
-  if ! err_file="$(mktemp -t launchd-unload-err)"; then
+  if ! err_file="$(mktemp -t launchd-unload-err.XXXXXX)"; then
     die "could not create a temp file to capture 'launchctl unload' stderr — cannot tell a real
         failure from a clean already-unloaded answer, so refusing to guess. The daemon's
         supervision state was NOT touched."
@@ -317,7 +332,7 @@ unload_launchd_if_loaded() {
 
 stop_systemd_if_loaded() {
   local err_file rc=0
-  if ! err_file="$(mktemp -t systemd-stop-err)"; then
+  if ! err_file="$(mktemp -t systemd-stop-err.XXXXXX)"; then
     die "could not create a temp file to capture 'systemctl --user stop' stderr — cannot tell a
         real failure from a clean already-stopped answer, so refusing to guess. The daemon's
         supervision state was NOT touched."
@@ -349,6 +364,14 @@ stop_systemd_if_loaded() {
 # "none" now means only: neither supervisor is installed, neither is loaded, and neither probe
 # errored.
 #
+# fleetd #545: *_ERRORED carries a THIRD state (2 = the probe's own mktemp setup failed, before
+# `systemctl` ever ran — see the flag's own comment above its initialization), and it must never be
+# reported with the same detail text as state 1 (`systemctl` ran and answered badly on stderr). The
+# two are different facts about different failures, and conflating them makes the "unclear" message
+# assert a cause ("systemctl exited non-zero and reported an error on stderr") that was never
+# measured when the real cause was state 2. detect_supervisor below picks the detail text off the
+# flag's value, not off a single "errored at all" boolean.
+#
 # fleetd #492 follow-up — constraints every caller of this function depends on (learned the hard
 # way: an earlier version of this fix set a SUPERVISOR_UNCLEAR_DETAIL global from inside here and
 # it was silently lost, because every real call site invokes this as `$(detect_supervisor)`):
@@ -382,7 +405,10 @@ detect_supervisor() {
   launchd_installed && li=1
   systemd_installed && si=1
 
-  if [ "$SYSTEMD_LOADED_ERRORED" = 1 ] || [ "$SYSTEMD_INSTALLED_ERRORED" = 1 ]; then
+  if [ "$SYSTEMD_LOADED_ERRORED" = 2 ] || [ "$SYSTEMD_INSTALLED_ERRORED" = 2 ]; then
+    detail="the systemd --user probe for '$SYSTEMD_UNIT' could not even be set up (a temp file to capture systemctl's stderr could not be created) — systemctl was never run, so this says nothing about systemd, the user bus, or the unit itself"
+    kind="unclear"
+  elif [ "$SYSTEMD_LOADED_ERRORED" = 1 ] || [ "$SYSTEMD_INSTALLED_ERRORED" = 1 ]; then
     detail="the systemd --user probe for '$SYSTEMD_UNIT' could not answer cleanly (systemctl exited non-zero and reported an error on stderr, not a clean negative — e.g. it cannot reach the user bus)"
     kind="unclear"
   elif [ "$ld" = 1 ] && [ "$sd" = 1 ]; then
@@ -830,7 +856,7 @@ if [ "$DO_BUILD" = 1 ]; then
   # fleetd #493: wipe a leftover staged jar from a previous failed/interrupted run BEFORE doing
   # anything else, so that run's leftovers can never be mistaken for this run's output.
   rm -f "$JAR_STAGED"
-  BUILD_LOG="$(mktemp -t fleetd-build)"
+  BUILD_LOG="$(mktemp -t fleetd-build.XXXXXX)"
   echo "   log: $BUILD_LOG"
   if ! mvn -f "$MODULE/pom.xml" clean install > "$BUILD_LOG" 2>&1; then
     grep -E 'ERROR|BUILD FAILURE|Tests run:.*Failures: [1-9]|Tests run:.*Errors: [1-9]' "$BUILD_LOG" \
@@ -1060,7 +1086,7 @@ tail -n "+$((RESTART_MARK + 1))" "$OUT" 2>/dev/null \
 
 # Errors since the restart, anchored to the marker so old noise cannot leak in. Keep the fresh
 # region in a file because the classifier must preserve the order of errors and recoveries.
-FRESH_LOG="$(mktemp -t fleetd-fresh-log)"
+FRESH_LOG="$(mktemp -t fleetd-fresh-log.XXXXXX)"
 trap 'rm -f "$FRESH_LOG"' EXIT
 tail -n "+$((RESTART_MARK + 1))" "$OUT" > "$FRESH_LOG" 2>/dev/null || true
 classify_amqp_connection_errors "$FRESH_LOG"

scripts/test-redeploy-fleetd.sh

@@ -128,8 +128,85 @@ STUB
   launchd_loaded()    { return 1; }
   result="$(PATH="$bin_dir:$PATH" detect_supervisor)"
   assert_equals "unclear" "$(supervisor_kind_of "$result")" "a systemd probe error must read as unclear, not none"
-  printf '%s' "$(supervisor_detail_of "$result")" | grep -qF "$SYSTEMD_UNIT" \
+  local detail
+  detail="$(supervisor_detail_of "$result")"
+  printf '%s' "$detail" | grep -qF "$SYSTEMD_UNIT" \
     || fail "detail does not name the systemd unit whose probe errored"
+  # fleetd #545: this is the PROBE-RAN-AND-ANSWERED-BADLY case (systemctl actually executed and
+  # wrote to stderr) — it must carry that story and never the SET-UP-FAILED story (mktemp never
+  # even ran here), or the two "unclear" causes have collapsed back into one message that asserts a
+  # cause it did not measure, which is the exact defect this ticket exists to fix.
+  printf '%s' "$detail" | grep -qF "systemctl exited non-zero and reported an error on stderr" \
+    || fail "detail does not say systemctl ran and answered with stderr: $detail"
+  printf '%s' "$detail" | grep -qF "could not even be set up" \
+    && fail "detail wrongly claims the probe could not be set up, but systemctl actually ran and answered on stderr: $detail"
+  return 0
+}
+
+# fleetd #545 — the companion case to the probe-error test above: here `mktemp` itself fails
+# (whatever the reason — the historical bug was a GNU-mktemp-rejects-a-template-with-no-Xs case,
+# but this stub simulates ANY reason the probe's own stderr-capture temp file cannot be created,
+# e.g. a full or unwritable temp dir) and `systemctl` is never invoked at all. Before this ticket,
+# this collapsed into the SAME "systemctl exited non-zero and reported an error on stderr" detail
+# as the sibling test above, which asserts a cause (systemctl ran and answered badly) that was
+# never measured, because systemctl never ran. This proves the SET-UP-FAILED detail is distinct and
+# does not claim systemctl said anything.
+test_detect_supervisor_systemd_probe_setup_failure_is_unclear() {
+  # Re-source first for the same reason test_detect_supervisor_systemd_probe_error_is_unclear does:
+  # restore the REAL probe bodies before driving them through a stub PATH.
+  source "$ROOT/scripts/redeploy-fleetd.sh"
+  local bin_dir result rc=0
+  bin_dir="$TMP/stub-bin-mktemp-fails"
+  mkdir -p "$bin_dir"
+  # A systemctl stub that would fail loudly if it were ever actually invoked — proves the mktemp
+  # failure short-circuits the probe before systemctl runs, not merely that this test forgot to
+  # supply a working systemctl.
+  cat > "$bin_dir/systemctl" <<'STUB'
+#!/usr/bin/env bash
+echo "systemctl must never run when mktemp already failed" >&2
+exit 1
+STUB
+  chmod +x "$bin_dir/systemctl"
+  cat > "$bin_dir/mktemp" <<'STUB'
+#!/usr/bin/env bash
+echo "mktemp: cannot create temp file" >&2
+exit 1
+STUB
+  chmod +x "$bin_dir/mktemp"
+
+  PATH="$bin_dir:$PATH" systemd_loaded && rc=0 || rc=$?
+  [ "$rc" -ne 0 ] \
+    || fail "systemd_loaded must not report loaded=true when its own mktemp setup failed"
+  assert_equals "2" "$SYSTEMD_LOADED_ERRORED" \
+    "systemd_loaded must flag a SETUP failure (2), distinct from a probe-answered-with-stderr failure (1)"
+
+  launchd_installed() { return 1; }
+  launchd_loaded()    { return 1; }
+  result="$(PATH="$bin_dir:$PATH" detect_supervisor)"
+  assert_equals "unclear" "$(supervisor_kind_of "$result")" "a systemd probe setup failure must read as unclear, not none"
+  local detail
+  detail="$(supervisor_detail_of "$result")"
+  printf '%s' "$detail" | grep -qF "could not even be set up" \
+    || fail "detail does not say the probe could not be SET UP: $detail"
+  printf '%s' "$detail" | grep -qF "systemctl exited non-zero and reported an error on stderr" \
+    && fail "detail wrongly asserts systemctl exited non-zero and reported an error on stderr, but systemctl was never run: $detail"
+  return 0
+}
+
+# fleetd #545 — source-text check: every `mktemp -t` template in redeploy-fleetd.sh must contain an
+# `X` placeholder. BSD mktemp (macOS) tolerates a bare template with no `X`s and just appends its
+# own random suffix, which is exactly why six such sites survived undetected here — GNU mktemp
+# (every Linux distribution) refuses a template with fewer than three `X`s and exits non-zero. There
+# is no BSD-vs-GNU seam to stub on this Mac, so this is a source-text check rather than a
+# behavioural one, the same shape as test_refuse_drain_gate_call_site_present above. Anchored on
+# `mktemp -t ` (with the trailing space) so it inspects only the `-t`-style templates this ticket is
+# about, never the `mktemp -d` calls this file and test-probe-member-credentials.sh already use
+# (both already carry their own `XXXXXX` and are a different mktemp mode entirely).
+test_mktemp_dash_t_templates_have_x_placeholders() {
+  local src="$ROOT/scripts/redeploy-fleetd.sh" bad
+  bad="$(grep -n 'mktemp -t ' "$src" | grep -v 'XXX' || true)"
+  [ -z "$bad" ] \
+    || fail "mktemp -t template(s) with no X placeholder (fails under GNU coreutils): $bad"
 }
 
 # fleetd #492 follow-up (Item 1): this must go through the REAL call-site shape at :437-440, not a
@@ -1088,6 +1165,8 @@ test_detect_supervisor_none
 test_detect_supervisor_systemd_installed_not_loaded_is_unclear
 test_detect_supervisor_launchd_installed_not_loaded_is_unclear
 test_detect_supervisor_systemd_probe_error_is_unclear
+test_detect_supervisor_systemd_probe_setup_failure_is_unclear
+test_mktemp_dash_t_templates_have_x_placeholders
 test_require_drivable_supervisor_refuses_ambiguous
 test_require_drivable_supervisor_refuses_unclear
 test_require_drivable_supervisor_accepts_known_kinds