CB-578 stage B: quarantine the exhausted credential, not the profile

A BACKEND_EXHAUSTED classification (stage A) now puts that profile's credential into a BackendQuarantine for a configurable cooldown. A spawn onto a quarantined profile is refused naming the credential and roughly when it lifts; weighted/round-robin/fixed placement skip a quarantined candidate; the quarantine lifts itself on the injected clock; and it is visible on bridge_profiles. Keyed by credential, not by profile name, via the new Profile.credentialId (profiles sharing one credential quarantine together — e.g. two models on one account) and effectiveCredentialId() (unset ⇒ quarantines alone, today's behaviour unchanged). Fixed a related gap along the way: a reload changing exhaustedPattern was silently reported "applied" even though it's deferred — sameLaunchSettings() now catches it too.
2026-08-15 10:29:54 +02:00
parent c00a86b32c
commit e501d39988
20 changed files with 886 additions and 79 deletions
@@ -0,0 +1,108 @@
+package dev.ltms.bridged.placement;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+import java.util.OptionalLong;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * CB-578 stage B: the credential-keyed quarantine tracker itself, isolated from placement/spawn
+ * wiring (that's {@code CompositePeerLauncherTest}). The clock is a plain {@link AtomicLong} of
+ * nanos so expiry is exercised without a real sleep.
+ */
+class BackendQuarantineTest {
+
+    @Test
+    void aFreshCredentialIsNotQuarantined() {
+        BackendQuarantine q = new BackendQuarantine(() -> 0L, TimeUnit.MINUTES.toNanos(30));
+        assertFalse(q.isQuarantined("shared-openai"));
+        assertEquals(OptionalLong.empty(), q.remainingSeconds("shared-openai"));
+    }
+
+    @Test
+    void quarantineBlocksTheCredentialForTheFullCooldown() {
+        BackendQuarantine q = new BackendQuarantine(() -> 0L, TimeUnit.MINUTES.toNanos(30));
+        q.quarantine("shared-openai");
+
+        assertTrue(q.isQuarantined("shared-openai"));
+        assertEquals(OptionalLong.of(1800L), q.remainingSeconds("shared-openai"));
+    }
+
+    @Test
+    void onlyTheQuarantinedCredentialIsAffected() {
+        BackendQuarantine q = new BackendQuarantine(() -> 0L, TimeUnit.MINUTES.toNanos(30));
+        q.quarantine("shared-openai");
+
+        assertFalse(q.isQuarantined("some-other-credential"),
+                "an unrelated credential must not be swept into the quarantine");
+    }
+
+    @Test
+    void expiresOnTheInjectedClock() {
+        AtomicLong now = new AtomicLong(0L);
+        BackendQuarantine q = new BackendQuarantine(now::get, TimeUnit.MINUTES.toNanos(30));
+        q.quarantine("shared-openai");
+        assertTrue(q.isQuarantined("shared-openai"));
+
+        now.set(TimeUnit.MINUTES.toNanos(29));
+        assertTrue(q.isQuarantined("shared-openai"), "still inside the cooldown");
+
+        now.set(TimeUnit.MINUTES.toNanos(31));
+        assertFalse(q.isQuarantined("shared-openai"), "the cooldown has elapsed on the injected clock");
+        assertEquals(OptionalLong.empty(), q.remainingSeconds("shared-openai"));
+    }
+
+    @Test
+    void aRepeatQuarantineCallRestartsTheCooldownAtFullLength() {
+        AtomicLong now = new AtomicLong(0L);
+        BackendQuarantine q = new BackendQuarantine(now::get, TimeUnit.MINUTES.toNanos(30));
+        q.quarantine("shared-openai");
+
+        now.set(TimeUnit.MINUTES.toNanos(20));
+        q.quarantine("shared-openai");
+
+        now.set(TimeUnit.MINUTES.toNanos(45)); // 25 min after the second call, 45 after the first
+        assertTrue(q.isQuarantined("shared-openai"),
+                "a fresh exhaustion resets the cooldown to full length, not the earlier shorter wait");
+    }
+
+    @Test
+    void activeRemainingSecondsListsOnlyStillQuarantinedCredentials() {
+        AtomicLong now = new AtomicLong(0L);
+        BackendQuarantine q = new BackendQuarantine(now::get, TimeUnit.MINUTES.toNanos(30));
+        q.quarantine("shared-openai");
+        q.quarantine("another-credential");
+
+        now.set(TimeUnit.MINUTES.toNanos(31));
+        q.quarantine("shared-openai"); // re-quarantined after the first one expired
+
+        Map<String, Long> active = q.activeRemainingSeconds();
+        assertEquals(Map.of("shared-openai", 1800L), active,
+                "the expired credential is dropped; the re-quarantined one is reported");
+    }
+
+    @Test
+    void noneReportsNothingQuarantinedWhenNeverToldTo() {
+        // .none() is the stand-in for a caller whose code path never calls #quarantine at all (e.g.
+        // the 2/5/6-arg CompositePeerLauncher constructors) — not a guarantee that a call to
+        // #quarantine on it is a no-op. Left alone, as those call sites leave it, nothing is ever
+        // quarantined.
+        BackendQuarantine q = BackendQuarantine.none();
+
+        assertFalse(q.isQuarantined("anything"));
+        assertTrue(q.activeRemainingSeconds().isEmpty());
+    }
+
+    @Test
+    void aNonPositiveCooldownIsRejected() {
+        assertThrows(IllegalArgumentException.class, () -> new BackendQuarantine(() -> 0L, 0L));
+        assertThrows(IllegalArgumentException.class, () -> new BackendQuarantine(() -> 0L, -1L));
+    }
+}