switchroom 0.15.2 → 0.15.3

package/dist/auth-broker/index.js

@@ -12363,6 +12363,47 @@ function resolveConsumerProbeIntervalMs(env) {
   return DEFAULT_CONSUMER_PROBE_INTERVAL_MS;
 }
 
+// src/auth/broker/account-eligibility.ts
+var WALL_PCT = 99.5;
+var HEALTHY_CLEAR_PCT = 80;
+var SNAPSHOT_STALE_AGE_MS = 24 * 60 * 60 * 1000;
+function snapshotFresh(s, now, maxAgeMs = SNAPSHOT_STALE_AGE_MS) {
+  return !!s && now - s.capturedAt <= maxAgeMs && s.capturedAt <= now + 60000;
+}
+function snapshotWalled(s) {
+  return s.fiveHourUtilizationPct >= WALL_PCT || s.sevenDayUtilizationPct >= WALL_PCT;
+}
+function snapshotClearlyHealthy(s) {
+  return s.fiveHourUtilizationPct < HEALTHY_CLEAR_PCT && s.sevenDayUtilizationPct < HEALTHY_CLEAR_PCT;
+}
+function isAccountBlocked(opts) {
+  const { mark, snapshot, now } = opts;
+  if (snapshotFresh(snapshot, now)) {
+    const markedAt = mark?.marked_at ?? 0;
+    if (snapshot.capturedAt >= markedAt) {
+      return snapshotWalled(snapshot);
+    }
+  }
+  return mark !== undefined && mark.exhausted_until > now;
+}
+function snapshotShouldClearMark(snapshot, mark, now) {
+  if (!mark)
+    return false;
+  if (!snapshotFresh(snapshot, now))
+    return false;
+  if (snapshot.capturedAt < (mark.marked_at ?? 0))
+    return false;
+  return snapshotClearlyHealthy(snapshot);
+}
+function clampMarkExpiry(opts) {
+  const { proposedUntil, now, shortMs, snapshot } = opts;
+  const shortCeil = now + shortMs;
+  if (proposedUntil <= shortCeil)
+    return proposedUntil;
+  const liveContradictsWeeklyWall = snapshotFresh(snapshot, now) && snapshot.sevenDayUtilizationPct < WALL_PCT;
+  return liveContradictsWeeklyWall ? shortCeil : proposedUntil;
+}
+
 // src/util/atomic.ts
 import { randomBytes } from "node:crypto";
 import { closeSync, constants, fsyncSync, openSync, renameSync, rmSync, writeSync } from "node:fs";
@@ -14030,8 +14071,11 @@ class AuthBroker {
     return this.accountWithFailover(account);
   }
   isAccountExhausted(account) {
-    const q = this.quota[account];
-    return q !== undefined && q.exhausted_until > this.now();
+    return isAccountBlocked({
+      mark: this.quota[account],
+      snapshot: this.lastQuotaCache[account],
+      now: this.now()
+    });
   }
   accountWithFailover(account) {
     if (!account || !this.isAccountExhausted(account))
@@ -14067,7 +14111,7 @@ class AuthBroker {
       const creds = readAccountCredentials(label, this.home);
       const meta = readAccountMeta(label, this.home);
       const q = this.quota[label];
-      const exhausted = q !== undefined && q.exhausted_until > this.now();
+      const exhausted = this.isAccountExhausted(label);
       const lq = this.lastQuotaCache[label];
       return {
         label,
@@ -14142,7 +14186,7 @@ class AuthBroker {
   cacheQuotaSnapshot(label, result) {
     if (!result.ok)
       return;
-    this.lastQuotaCache[label] = {
+    const snapshot = {
       fiveHourUtilizationPct: result.data.fiveHourUtilizationPct,
       sevenDayUtilizationPct: result.data.sevenDayUtilizationPct,
       fiveHourResetAt: result.data.fiveHourResetAt?.toISOString() ?? null,
@@ -14152,6 +14196,13 @@ class AuthBroker {
       overageDisabledReason: result.data.overageDisabledReason,
       capturedAt: this.now()
     };
+    this.lastQuotaCache[label] = snapshot;
+    if (snapshotShouldClearMark(snapshot, this.quota[label], this.now())) {
+      delete this.quota[label];
+      this.persistQuota();
+      process.stdout.write(`auth-broker: live probe shows ${label} healthy (5h=${snapshot.fiveHourUtilizationPct}% 7d=${snapshot.sevenDayUtilizationPct}%) — cleared stale exhaustion mark
+`);
+    }
   }
   async fleetQuotaProbeTick() {
     for (const label of listAccounts(this.home)) {
@@ -14183,14 +14234,21 @@ class AuthBroker {
         this.logErr(`consumer-quota-probe ${label}: ${err.message}`);
         continue;
       }
+      this.cacheQuotaSnapshot(label, result);
       const decision = quotaIndicatesExhaustion(result);
       if (!decision.exhausted)
         continue;
-      const exhaustedUntil = decision.until ?? this.now() + MARK_EXHAUSTED_DEFAULT_MS;
+      const now = this.now();
+      const exhaustedUntil = clampMarkExpiry({
+        proposedUntil: decision.until ?? now + MARK_EXHAUSTED_DEFAULT_MS,
+        now,
+        shortMs: MARK_EXHAUSTED_DEFAULT_MS,
+        snapshot: this.lastQuotaCache[label]
+      });
       const existing = this.quota[label]?.exhausted_until;
       if (existing !== undefined && existing >= exhaustedUntil)
         continue;
-      this.quota[label] = { exhausted_until: exhaustedUntil };
+      this.quota[label] = { exhausted_until: exhaustedUntil, marked_at: now };
       this.persistQuota();
       this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, ok: true });
       process.stdout.write(`auth-broker: consumer-quota-sensor marked ${label} exhausted until ${new Date(exhaustedUntil).toISOString()} — consumer(s) fail over
@@ -14224,8 +14282,14 @@ class AuthBroker {
       socket.write(encodeError(id, "ACCOUNT_NOT_FOUND", "no active account configured"));
       return;
     }
-    const exhaustedUntil = until ?? this.now() + MARK_EXHAUSTED_DEFAULT_MS;
-    this.quota[account] = { exhausted_until: exhaustedUntil };
+    const now = this.now();
+    const exhaustedUntil = clampMarkExpiry({
+      proposedUntil: until ?? now + MARK_EXHAUSTED_DEFAULT_MS,
+      now,
+      shortMs: MARK_EXHAUSTED_DEFAULT_MS,
+      snapshot: this.lastQuotaCache[account]
+    });
+    this.quota[account] = { exhausted_until: exhaustedUntil, marked_at: now };
     this.persistQuota();
     const rolled = this.fanoutFailoverFor(account);
     const rolledTo = this.nextHealthyAccount(account, this.config.auth?.fallback_order ?? []);
@@ -14752,9 +14816,7 @@ class AuthBroker {
       const cand = order[(start + i) % order.length];
       if (!cand)
         continue;
-      const q = this.quota[cand];
-      const exhausted = q !== undefined && q.exhausted_until > this.now();
-      if (!exhausted && accountExists(cand, this.home))
+      if (!this.isAccountExhausted(cand) && accountExists(cand, this.home))
         return cand;
     }
     return null;