switchroom 0.18.9 → 0.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +198 -13
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/switchroom.js +214 -83
- package/dist/cli/ui/apple-touch-icon.png +0 -0
- package/dist/cli/ui/favicon-32.png +0 -0
- package/dist/cli/ui/favicon.ico +0 -0
- package/dist/host-control/main.js +760 -257
- package/dist/vault/approvals/kernel-server.js +3 -1
- package/dist/vault/broker/server.js +3 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +119 -37
- package/profiles/_shared/dev-protocol.md.hbs +42 -0
- package/skills/dev-protocol/SKILL.md +131 -0
- package/telegram-plugin/README.md +2 -1
- package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
- package/telegram-plugin/admin-commands/index.ts +6 -1
- package/telegram-plugin/bridge/bridge.ts +23 -1
- package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
- package/telegram-plugin/chat-lock.ts +13 -0
- package/telegram-plugin/dist/bridge/bridge.js +24 -1
- package/telegram-plugin/dist/gateway/gateway.js +1832 -263
- package/telegram-plugin/dist/server.js +29 -2
- package/telegram-plugin/fallback-card-collapse.ts +131 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
- package/telegram-plugin/gateway/effort-command.ts +47 -3
- package/telegram-plugin/gateway/gateway.ts +1435 -211
- package/telegram-plugin/gateway/model-command.ts +94 -8
- package/telegram-plugin/gateway/pending-session-command.ts +365 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
- package/telegram-plugin/gateway/session-model-file.ts +166 -23
- package/telegram-plugin/gateway/stop-command.ts +56 -0
- package/telegram-plugin/photo-precheck.ts +201 -0
- package/telegram-plugin/quota-watch.ts +141 -2
- package/telegram-plugin/registry/subagents-schema.ts +26 -3
- package/telegram-plugin/registry/subagents.test.ts +67 -0
- package/telegram-plugin/retry-api-call.ts +31 -0
- package/telegram-plugin/subagent-watcher.ts +392 -1
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
- package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
- package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
- package/telegram-plugin/tests/effort-command.test.ts +59 -2
- package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
- package/telegram-plugin/tests/model-command.test.ts +46 -3
- package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
- package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
- package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
- package/telegram-plugin/tests/quota-watch.test.ts +225 -0
- package/telegram-plugin/tests/session-model-file.test.ts +101 -2
- package/telegram-plugin/tests/stop-command.test.ts +234 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
- package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
- package/telegram-plugin/welcome-text.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +27 -0
|
@@ -11646,6 +11646,7 @@ var SwitchroomConfigSchema = exports_external.object({
|
|
|
11646
11646
|
uid: exports_external.number().int().nonnegative().optional().describe("Optional UID to chown the consumer socket to (defaults to 0 = root, " + "suitable for sibling containers running as root)."),
|
|
11647
11647
|
mirror_dir: exports_external.string().optional().describe("Optional host-side directory path. When set, the broker actively " + "writes the consumer's effective-account `.credentials.json` mirror " + "here — in addition to serving creds on demand via `get-credentials`. " + "Use this to eliminate the pull-latency gap: without a mirror the " + "consumer only gets failover creds at its next scheduled re-fetch " + "(up to 30 min). With a mirror the broker pushes failover creds " + "immediately when it detects exhaustion (consumer-quota-sensor tick, " + "or a mark-exhausted RPC on the pinned account). The directory must " + "be accessible to the broker container (bind-mounted from the host) " + "and to the consumer container; the broker writes " + "`<mirror_dir>/.credentials.json` atomically. Chown is attempted to " + "`uid` (default 0) — swallowed when CAP_CHOWN is absent.")
|
|
11648
11648
|
})).optional().describe("Non-agent peers that hold a broker socket (RFC H §4.8). Each gets " + "its own `/run/switchroom/auth-broker/<name>/sock` chowned to its UID. " + "Consumers cannot be admins; a consumer name that collides with an " + "agent (whether that agent has `admin: true` or not) is a config " + "error caught at schema validation."),
|
|
11649
|
+
proactive_failover_pct: exports_external.number().min(1).max(99).optional().describe("Soft-avoid threshold (percent) for proactive auth failover (#3031). " + "When set (e.g. 95), an account whose fresh 7-day utilization is at/" + "above this value — or whose 5h utilization is at/above min(pct+3, " + "98) — enters the SOFT-AVOID preference tier: the broker prefers a " + "fully-eligible fallback account on the serving path, without ever " + "blocking the account (the hard wall stays 99.5). Enter/exit uses " + "hysteresis (exit below pct-5 or on window reset) so the preference " + "does not flap between probe ticks. Accounts serving past the wall " + "via `allow_overage_accounts` are never soft-avoided. UNSET (the " + "default) disables the tier entirely — behavior is identical to a " + "fleet without this field."),
|
|
11649
11650
|
allow_overage_accounts: exports_external.array(exports_external.string().min(1)).optional().describe("Opt-in list of account labels (bare strings matching `auth.active` / " + "`auth.fallback_order` entries) that may be served PAST the weekly " + "utilization wall when Anthropic overage billing is available for the " + "account (`overageStatus === 'allowed'`). Overage is REAL MONEY — " + "default is empty (no account gets this). An account in this list is " + "only kept eligible when its fresh quota snapshot reports " + "`overageStatus: 'allowed'` AND `overageDisabledReason` is NOT " + "'out_of_credits' (i.e. the overage credit has not been exhausted). " + "As soon as `overageDisabledReason` becomes 'out_of_credits', the " + "account is blocked immediately regardless of this flag. Overage lifts " + "ONLY the utilization wall — it cannot lift an active exhaustion mark " + "written by a real 429 (`mark-exhausted`).")
|
|
11650
11651
|
}).optional().describe("Switchroom-auth-broker configuration (RFC H). Fleet-wide active account, " + "fallback order, admin-agent ACL, and ephemeral-consumer surface. " + "Required from the v0.8+ schema onwards; pre-v0.8 fleets are migrated " + "in-place by `switchroom apply` (see src/auth/migrate-schema.ts)."),
|
|
11651
11652
|
drive: GoogleWorkspaceConfigSchema.describe("RFC D legacy key — use `google_workspace:` instead. Optional Google " + "Workspace onboarding configuration. When set, supplies Google OAuth " + "client credentials, the approver allowlist for `switchroom drive " + "connect`, and the optional tier knob. Env vars " + "(SWITCHROOM_GOOGLE_CLIENT_ID, SWITCHROOM_GOOGLE_CLIENT_SECRET, " + "SWITCHROOM_APPROVER_USER_ID) take precedence over this block when " + "set, preserving back-compat with the env-only flow shipped in #766."),
|
|
@@ -17275,6 +17275,7 @@ var SwitchroomConfigSchema = exports_external.object({
|
|
|
17275
17275
|
uid: exports_external.number().int().nonnegative().optional().describe("Optional UID to chown the consumer socket to (defaults to 0 = root, " + "suitable for sibling containers running as root)."),
|
|
17276
17276
|
mirror_dir: exports_external.string().optional().describe("Optional host-side directory path. When set, the broker actively " + "writes the consumer's effective-account `.credentials.json` mirror " + "here — in addition to serving creds on demand via `get-credentials`. " + "Use this to eliminate the pull-latency gap: without a mirror the " + "consumer only gets failover creds at its next scheduled re-fetch " + "(up to 30 min). With a mirror the broker pushes failover creds " + "immediately when it detects exhaustion (consumer-quota-sensor tick, " + "or a mark-exhausted RPC on the pinned account). The directory must " + "be accessible to the broker container (bind-mounted from the host) " + "and to the consumer container; the broker writes " + "`<mirror_dir>/.credentials.json` atomically. Chown is attempted to " + "`uid` (default 0) — swallowed when CAP_CHOWN is absent.")
|
|
17277
17277
|
})).optional().describe("Non-agent peers that hold a broker socket (RFC H §4.8). Each gets " + "its own `/run/switchroom/auth-broker/<name>/sock` chowned to its UID. " + "Consumers cannot be admins; a consumer name that collides with an " + "agent (whether that agent has `admin: true` or not) is a config " + "error caught at schema validation."),
|
|
17278
|
+
proactive_failover_pct: exports_external.number().min(1).max(99).optional().describe("Soft-avoid threshold (percent) for proactive auth failover (#3031). " + "When set (e.g. 95), an account whose fresh 7-day utilization is at/" + "above this value — or whose 5h utilization is at/above min(pct+3, " + "98) — enters the SOFT-AVOID preference tier: the broker prefers a " + "fully-eligible fallback account on the serving path, without ever " + "blocking the account (the hard wall stays 99.5). Enter/exit uses " + "hysteresis (exit below pct-5 or on window reset) so the preference " + "does not flap between probe ticks. Accounts serving past the wall " + "via `allow_overage_accounts` are never soft-avoided. UNSET (the " + "default) disables the tier entirely — behavior is identical to a " + "fleet without this field."),
|
|
17278
17279
|
allow_overage_accounts: exports_external.array(exports_external.string().min(1)).optional().describe("Opt-in list of account labels (bare strings matching `auth.active` / " + "`auth.fallback_order` entries) that may be served PAST the weekly " + "utilization wall when Anthropic overage billing is available for the " + "account (`overageStatus === 'allowed'`). Overage is REAL MONEY — " + "default is empty (no account gets this). An account in this list is " + "only kept eligible when its fresh quota snapshot reports " + "`overageStatus: 'allowed'` AND `overageDisabledReason` is NOT " + "'out_of_credits' (i.e. the overage credit has not been exhausted). " + "As soon as `overageDisabledReason` becomes 'out_of_credits', the " + "account is blocked immediately regardless of this flag. Overage lifts " + "ONLY the utilization wall — it cannot lift an active exhaustion mark " + "written by a real 429 (`mark-exhausted`).")
|
|
17279
17280
|
}).optional().describe("Switchroom-auth-broker configuration (RFC H). Fleet-wide active account, " + "fallback order, admin-agent ACL, and ephemeral-consumer surface. " + "Required from the v0.8+ schema onwards; pre-v0.8 fleets are migrated " + "in-place by `switchroom apply` (see src/auth/migrate-schema.ts)."),
|
|
17280
17281
|
drive: GoogleWorkspaceConfigSchema.describe("RFC D legacy key — use `google_workspace:` instead. Optional Google " + "Workspace onboarding configuration. When set, supplies Google OAuth " + "client credentials, the approver allowlist for `switchroom drive " + "connect`, and the optional tier knob. Env vars " + "(SWITCHROOM_GOOGLE_CLIENT_ID, SWITCHROOM_GOOGLE_CLIENT_SECRET, " + "SWITCHROOM_APPROVER_USER_ID) take precedence over this block when " + "set, preserving back-compat with the env-only flow shipped in #766."),
|
|
@@ -18135,7 +18136,7 @@ import_handlebars.default.registerHelper("isNumber", (value) => {
|
|
|
18135
18136
|
return typeof value === "number" && Number.isFinite(value);
|
|
18136
18137
|
});
|
|
18137
18138
|
var SHARED_FRAGMENTS_DIR = resolve4(PROFILES_ROOT, "_shared");
|
|
18138
|
-
var SHARED_FRAGMENTS = ["vault-protocol", "agent-self-service", "execution-discipline", "reply-discipline"];
|
|
18139
|
+
var SHARED_FRAGMENTS = ["vault-protocol", "agent-self-service", "execution-discipline", "reply-discipline", "dev-protocol"];
|
|
18139
18140
|
for (const name of SHARED_FRAGMENTS) {
|
|
18140
18141
|
const fragPath = join2(SHARED_FRAGMENTS_DIR, `${name}.md.hbs`);
|
|
18141
18142
|
if (existsSync4(fragPath)) {
|
|
@@ -18754,6 +18755,51 @@ function clampMarkExpiry(opts) {
|
|
|
18754
18755
|
const liveContradictsWeeklyWall = snapshotFresh(snapshot, now) && snapshot.sevenDayUtilizationPct < WALL_PCT;
|
|
18755
18756
|
return liveContradictsWeeklyWall ? shortCeil : proposedUntil;
|
|
18756
18757
|
}
|
|
18758
|
+
var SOFT_AVOID_FIVE_HOUR_CAP = 98;
|
|
18759
|
+
var SOFT_AVOID_EXIT_MARGIN = 5;
|
|
18760
|
+
function softAvoidThresholds(pct) {
|
|
18761
|
+
return { sevenDay: pct, fiveHour: Math.min(pct + 3, SOFT_AVOID_FIVE_HOUR_CAP) };
|
|
18762
|
+
}
|
|
18763
|
+
function evaluateSoftAvoid(opts) {
|
|
18764
|
+
const { snapshot, now, pct, overageLifted = false, prev } = opts;
|
|
18765
|
+
if (pct === undefined)
|
|
18766
|
+
return { softAvoided: false };
|
|
18767
|
+
if (overageLifted)
|
|
18768
|
+
return { softAvoided: false };
|
|
18769
|
+
if (!snapshotFresh(snapshot, now)) {
|
|
18770
|
+
return prev ?? { softAvoided: false };
|
|
18771
|
+
}
|
|
18772
|
+
const s = snapshot;
|
|
18773
|
+
const thresholds = softAvoidThresholds(pct);
|
|
18774
|
+
const fiveRefilled = s.fiveHourResetAtMs != null && s.fiveHourResetAtMs <= now;
|
|
18775
|
+
const sevenRefilled = s.sevenDayResetAtMs != null && s.sevenDayResetAtMs <= now;
|
|
18776
|
+
const five = fiveRefilled ? 0 : s.fiveHourUtilizationPct;
|
|
18777
|
+
const seven = sevenRefilled ? 0 : s.sevenDayUtilizationPct;
|
|
18778
|
+
const nextState = (softAvoided, triggeredBy) => ({
|
|
18779
|
+
softAvoided,
|
|
18780
|
+
...softAvoided && triggeredBy ? { triggeredBy } : {},
|
|
18781
|
+
fiveHourResetAt: s.fiveHourResetAtMs ?? prev?.fiveHourResetAt ?? null,
|
|
18782
|
+
sevenDayResetAt: s.sevenDayResetAtMs ?? prev?.sevenDayResetAt ?? null
|
|
18783
|
+
});
|
|
18784
|
+
const fiveTriggered = five >= thresholds.fiveHour;
|
|
18785
|
+
const sevenTriggered = seven >= thresholds.sevenDay;
|
|
18786
|
+
if (fiveTriggered || sevenTriggered) {
|
|
18787
|
+
const prevTrig = prev?.softAvoided ? prev.triggeredBy ?? { fiveHour: true, sevenDay: true } : undefined;
|
|
18788
|
+
return nextState(true, {
|
|
18789
|
+
fiveHour: fiveTriggered || (prevTrig?.fiveHour ?? false),
|
|
18790
|
+
sevenDay: sevenTriggered || (prevTrig?.sevenDay ?? false)
|
|
18791
|
+
});
|
|
18792
|
+
}
|
|
18793
|
+
if (prev?.softAvoided) {
|
|
18794
|
+
const trig = prev.triggeredBy ?? { fiveHour: true, sevenDay: true };
|
|
18795
|
+
const fiveReleased = !trig.fiveHour || fiveRefilled || prev.fiveHourResetAt != null && s.fiveHourResetAtMs != null && s.fiveHourResetAtMs > prev.fiveHourResetAt || five < thresholds.fiveHour - SOFT_AVOID_EXIT_MARGIN;
|
|
18796
|
+
const sevenReleased = !trig.sevenDay || sevenRefilled || prev.sevenDayResetAt != null && s.sevenDayResetAtMs != null && s.sevenDayResetAtMs > prev.sevenDayResetAt || seven < thresholds.sevenDay - SOFT_AVOID_EXIT_MARGIN;
|
|
18797
|
+
if (fiveReleased && sevenReleased)
|
|
18798
|
+
return nextState(false);
|
|
18799
|
+
return nextState(true, trig);
|
|
18800
|
+
}
|
|
18801
|
+
return nextState(false);
|
|
18802
|
+
}
|
|
18757
18803
|
|
|
18758
18804
|
// src/auth/broker/consumer-quota-sensor.ts
|
|
18759
18805
|
var EXHAUSTION_PCT = 99.5;
|
|
@@ -20017,6 +20063,7 @@ class AuthBroker {
|
|
|
20017
20063
|
shaIndex = {};
|
|
20018
20064
|
thresholdViolations = {};
|
|
20019
20065
|
notificationClaims = {};
|
|
20066
|
+
lastFleetRoll = null;
|
|
20020
20067
|
lastWrittenExpiresAt = new Map;
|
|
20021
20068
|
refreshInFlight = new Set;
|
|
20022
20069
|
consumerLastSeen = {};
|
|
@@ -20473,6 +20520,41 @@ class AuthBroker {
|
|
|
20473
20520
|
allowOverage: this.isOverageAllowed(account)
|
|
20474
20521
|
});
|
|
20475
20522
|
}
|
|
20523
|
+
softAvoidState = {};
|
|
20524
|
+
softAvoidRolledTo = {};
|
|
20525
|
+
softAvoidSnapshotOf(account) {
|
|
20526
|
+
const s = this.lastQuotaCache[account];
|
|
20527
|
+
if (!s)
|
|
20528
|
+
return;
|
|
20529
|
+
return {
|
|
20530
|
+
...s,
|
|
20531
|
+
fiveHourResetAtMs: s.fiveHourResetAt ? Date.parse(s.fiveHourResetAt) : null,
|
|
20532
|
+
sevenDayResetAtMs: s.sevenDayResetAt ? Date.parse(s.sevenDayResetAt) : null
|
|
20533
|
+
};
|
|
20534
|
+
}
|
|
20535
|
+
isAccountSoftAvoided(account) {
|
|
20536
|
+
const pct = this.config.auth?.proactive_failover_pct;
|
|
20537
|
+
if (pct === undefined)
|
|
20538
|
+
return false;
|
|
20539
|
+
const now = this.now();
|
|
20540
|
+
const snapshot = this.softAvoidSnapshotOf(account);
|
|
20541
|
+
const overageLifted = !!snapshot && snapshotFresh(snapshot, now) && overageLiftsWall(snapshot, this.isOverageAllowed(account));
|
|
20542
|
+
const state = evaluateSoftAvoid({
|
|
20543
|
+
snapshot,
|
|
20544
|
+
now,
|
|
20545
|
+
pct,
|
|
20546
|
+
overageLifted,
|
|
20547
|
+
prev: this.softAvoidState[account]
|
|
20548
|
+
});
|
|
20549
|
+
this.softAvoidState[account] = state;
|
|
20550
|
+
return state.softAvoided;
|
|
20551
|
+
}
|
|
20552
|
+
softAvoidUtilScore(account) {
|
|
20553
|
+
const s = this.lastQuotaCache[account];
|
|
20554
|
+
if (!s || !snapshotFresh(s, this.now()))
|
|
20555
|
+
return Number.POSITIVE_INFINITY;
|
|
20556
|
+
return Math.max(s.fiveHourUtilizationPct, s.sevenDayUtilizationPct);
|
|
20557
|
+
}
|
|
20476
20558
|
isActiveOverageServing(account) {
|
|
20477
20559
|
if (!account)
|
|
20478
20560
|
return false;
|
|
@@ -20519,8 +20601,9 @@ class AuthBroker {
|
|
|
20519
20601
|
}
|
|
20520
20602
|
async nextHealthyAccountLive(current, order) {
|
|
20521
20603
|
const cached = this.nextHealthyAccount(current, order);
|
|
20522
|
-
if (cached && this.accountEligibilityOf(cached) === "eligible")
|
|
20604
|
+
if (cached && this.accountEligibilityOf(cached) === "eligible" && !this.isAccountSoftAvoided(cached)) {
|
|
20523
20605
|
return cached;
|
|
20606
|
+
}
|
|
20524
20607
|
const start = order.indexOf(current);
|
|
20525
20608
|
const ring = start === -1 ? [...order] : order.map((_, i) => order[(start + 1 + i) % order.length]).filter((x) => !!x);
|
|
20526
20609
|
const unknowns = ring.filter((cand) => cand && cand !== current && accountExists(cand, this.home) && this.accountEligibilityOf(cand) === "unknown");
|
|
@@ -20536,14 +20619,46 @@ class AuthBroker {
|
|
|
20536
20619
|
return null;
|
|
20537
20620
|
}
|
|
20538
20621
|
accountWithFailover(account) {
|
|
20539
|
-
if (!account
|
|
20540
|
-
return
|
|
20622
|
+
if (!account)
|
|
20623
|
+
return null;
|
|
20624
|
+
const exhausted = this.isAccountExhausted(account);
|
|
20625
|
+
if (!exhausted && !this.isAccountSoftAvoided(account))
|
|
20626
|
+
return account;
|
|
20627
|
+
let softAvoidExcludedAny = false;
|
|
20541
20628
|
for (const cand of this.config.auth?.fallback_order ?? []) {
|
|
20542
20629
|
if (cand === account || this.isAccountExhausted(cand))
|
|
20543
20630
|
continue;
|
|
20631
|
+
if (this.isAccountSoftAvoided(cand)) {
|
|
20632
|
+
softAvoidExcludedAny = true;
|
|
20633
|
+
continue;
|
|
20634
|
+
}
|
|
20544
20635
|
if (readAccountCredentials(cand, this.home))
|
|
20545
20636
|
return cand;
|
|
20546
20637
|
}
|
|
20638
|
+
if (!exhausted) {
|
|
20639
|
+
let best = account;
|
|
20640
|
+
let bestScore = this.softAvoidUtilScore(account);
|
|
20641
|
+
for (const cand of this.config.auth?.fallback_order ?? []) {
|
|
20642
|
+
if (cand === account || this.isAccountExhausted(cand))
|
|
20643
|
+
continue;
|
|
20644
|
+
if (!readAccountCredentials(cand, this.home))
|
|
20645
|
+
continue;
|
|
20646
|
+
const score = this.softAvoidUtilScore(cand);
|
|
20647
|
+
if (score < bestScore) {
|
|
20648
|
+
best = cand;
|
|
20649
|
+
bestScore = score;
|
|
20650
|
+
}
|
|
20651
|
+
}
|
|
20652
|
+
return best;
|
|
20653
|
+
}
|
|
20654
|
+
if (softAvoidExcludedAny) {
|
|
20655
|
+
for (const cand of this.config.auth?.fallback_order ?? []) {
|
|
20656
|
+
if (cand === account || this.isAccountExhausted(cand))
|
|
20657
|
+
continue;
|
|
20658
|
+
if (readAccountCredentials(cand, this.home))
|
|
20659
|
+
return cand;
|
|
20660
|
+
}
|
|
20661
|
+
}
|
|
20547
20662
|
const active = this.config.auth?.active;
|
|
20548
20663
|
if (active && active !== account && !this.isAccountExhausted(active) && readAccountCredentials(active, this.home)) {
|
|
20549
20664
|
return active;
|
|
@@ -20603,7 +20718,8 @@ class AuthBroker {
|
|
|
20603
20718
|
accounts,
|
|
20604
20719
|
agents,
|
|
20605
20720
|
consumers,
|
|
20606
|
-
active_overage_serving
|
|
20721
|
+
active_overage_serving,
|
|
20722
|
+
last_fleet_roll: this.lastFleetRoll
|
|
20607
20723
|
}));
|
|
20608
20724
|
}
|
|
20609
20725
|
async opListGoogleAccounts(socket, id, identity) {
|
|
@@ -20698,6 +20814,7 @@ class AuthBroker {
|
|
|
20698
20814
|
}
|
|
20699
20815
|
}
|
|
20700
20816
|
async fleetQuotaProbeTick() {
|
|
20817
|
+
const probed = [];
|
|
20701
20818
|
for (const label of listAccounts(this.home)) {
|
|
20702
20819
|
const creds = readAccountCredentials(label, this.home);
|
|
20703
20820
|
const token = creds?.claudeAiOauth?.accessToken;
|
|
@@ -20711,16 +20828,65 @@ class AuthBroker {
|
|
|
20711
20828
|
continue;
|
|
20712
20829
|
}
|
|
20713
20830
|
this.cacheQuotaSnapshot(label, result);
|
|
20831
|
+
probed.push({ label, result });
|
|
20832
|
+
}
|
|
20833
|
+
for (const { label, result } of probed) {
|
|
20714
20834
|
const active = this.config.auth?.active;
|
|
20715
|
-
|
|
20835
|
+
const isPinnedAgentAccount = this.pinnedAgentAccounts().has(label);
|
|
20836
|
+
if (label !== active && !isPinnedAgentAccount)
|
|
20716
20837
|
continue;
|
|
20717
20838
|
const decision = quotaIndicatesExhaustion(result, this.isOverageAllowed(label));
|
|
20718
|
-
if (
|
|
20839
|
+
if (decision.exhausted) {
|
|
20840
|
+
process.stdout.write(`auth-broker: fleet-quota-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} exhausted — proactive failover
|
|
20841
|
+
`);
|
|
20842
|
+
this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true, reason: "hard-exhaustion" });
|
|
20843
|
+
const { rolledTo } = await this.markExhaustedAndRoll(label, decision.until ?? undefined, { kind: "operator" });
|
|
20844
|
+
if (rolledTo && label === active) {
|
|
20845
|
+
this.recordFleetRoll(label, rolledTo, "hard-exhaustion", decision.until ?? undefined);
|
|
20846
|
+
}
|
|
20719
20847
|
continue;
|
|
20720
|
-
|
|
20848
|
+
}
|
|
20849
|
+
this.softAvoidProbeRoll(label);
|
|
20850
|
+
}
|
|
20851
|
+
}
|
|
20852
|
+
recordFleetRoll(from, to, reason, exhaustedUntil) {
|
|
20853
|
+
const s = this.lastQuotaCache[from];
|
|
20854
|
+
this.lastFleetRoll = {
|
|
20855
|
+
from,
|
|
20856
|
+
to,
|
|
20857
|
+
at: this.now(),
|
|
20858
|
+
reason,
|
|
20859
|
+
...exhaustedUntil != null ? { exhausted_until: exhaustedUntil } : {},
|
|
20860
|
+
...s ? {
|
|
20861
|
+
window: s.fiveHourUtilizationPct >= s.sevenDayUtilizationPct ? "5h" : "7d",
|
|
20862
|
+
pct: Math.max(s.fiveHourUtilizationPct, s.sevenDayUtilizationPct)
|
|
20863
|
+
} : {}
|
|
20864
|
+
};
|
|
20865
|
+
this.persistLastFleetRoll();
|
|
20866
|
+
}
|
|
20867
|
+
pinnedAgentAccounts() {
|
|
20868
|
+
return new Set(Object.values(this.config.agents ?? {}).map((a) => a.auth?.override).filter((x) => typeof x === "string" && x.length > 0));
|
|
20869
|
+
}
|
|
20870
|
+
softAvoidProbeRoll(label) {
|
|
20871
|
+
if (!this.isAccountSoftAvoided(label)) {
|
|
20872
|
+
delete this.softAvoidRolledTo[label];
|
|
20873
|
+
return;
|
|
20874
|
+
}
|
|
20875
|
+
const target = this.accountWithFailover(label);
|
|
20876
|
+
if (!target || target === label || this.isAccountSoftAvoided(target)) {
|
|
20877
|
+
delete this.softAvoidRolledTo[label];
|
|
20878
|
+
return;
|
|
20879
|
+
}
|
|
20880
|
+
if (this.softAvoidRolledTo[label] === target)
|
|
20881
|
+
return;
|
|
20882
|
+
this.softAvoidRolledTo[label] = target;
|
|
20883
|
+
process.stdout.write(`auth-broker: fleet-quota-probe shows ${label} soft-avoided (≥ auth.proactive_failover_pct) — serving-preference roll → ${target}
|
|
20721
20884
|
`);
|
|
20722
|
-
|
|
20723
|
-
|
|
20885
|
+
this.audit({ op: "proactive-roll", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true, reason: "soft-avoid" });
|
|
20886
|
+
this.fanoutFailoverTo(label, target);
|
|
20887
|
+
this.fanoutToAffectedConsumers(label);
|
|
20888
|
+
if (label === this.config.auth?.active) {
|
|
20889
|
+
this.recordFleetRoll(label, target, "soft-avoid");
|
|
20724
20890
|
}
|
|
20725
20891
|
}
|
|
20726
20892
|
async consumerQuotaProbeTick() {
|
|
@@ -20893,6 +21059,8 @@ class AuthBroker {
|
|
|
20893
21059
|
socket.write(encodeError(id, "INTERNAL", err.message));
|
|
20894
21060
|
return;
|
|
20895
21061
|
}
|
|
21062
|
+
delete this.softAvoidState[label];
|
|
21063
|
+
delete this.softAvoidRolledTo[label];
|
|
20896
21064
|
this.chownAccountFilesIfRoot(label);
|
|
20897
21065
|
const contents = readFileSync7(accountCredentialsPath(label, this.home), "utf-8");
|
|
20898
21066
|
this.shaIndex[label] = sha256Hex(contents);
|
|
@@ -20934,6 +21102,8 @@ class AuthBroker {
|
|
|
20934
21102
|
delete this.thresholdViolations[label];
|
|
20935
21103
|
this.lastWrittenExpiresAt.delete(label);
|
|
20936
21104
|
delete this.lastQuotaCache[label];
|
|
21105
|
+
delete this.softAvoidState[label];
|
|
21106
|
+
delete this.softAvoidRolledTo[label];
|
|
20937
21107
|
this.persistShaIndex();
|
|
20938
21108
|
this.persistQuota();
|
|
20939
21109
|
this.persistThresholdViolations();
|
|
@@ -21415,14 +21585,23 @@ class AuthBroker {
|
|
|
21415
21585
|
const start = order.indexOf(current);
|
|
21416
21586
|
if (start === -1)
|
|
21417
21587
|
return order[0] ?? null;
|
|
21588
|
+
let bestSoftAvoided = null;
|
|
21589
|
+
let bestScore = Number.POSITIVE_INFINITY;
|
|
21418
21590
|
for (let i = 1;i <= order.length; i++) {
|
|
21419
21591
|
const cand = order[(start + i) % order.length];
|
|
21420
21592
|
if (!cand)
|
|
21421
21593
|
continue;
|
|
21422
|
-
if (
|
|
21594
|
+
if (this.isAccountExhausted(cand) || !accountExists(cand, this.home))
|
|
21595
|
+
continue;
|
|
21596
|
+
if (!this.isAccountSoftAvoided(cand))
|
|
21423
21597
|
return cand;
|
|
21598
|
+
const score = this.softAvoidUtilScore(cand);
|
|
21599
|
+
if (bestSoftAvoided === null || score < bestScore) {
|
|
21600
|
+
bestSoftAvoided = cand;
|
|
21601
|
+
bestScore = score;
|
|
21602
|
+
}
|
|
21424
21603
|
}
|
|
21425
|
-
return
|
|
21604
|
+
return bestSoftAvoided;
|
|
21426
21605
|
}
|
|
21427
21606
|
fanoutForAgent(name) {
|
|
21428
21607
|
const auth = this.config.auth ?? {};
|
|
@@ -21515,6 +21694,7 @@ class AuthBroker {
|
|
|
21515
21694
|
this.shaIndex = this.readJson("sha-index.json") ?? {};
|
|
21516
21695
|
this.thresholdViolations = this.readJson("threshold-violations.json") ?? {};
|
|
21517
21696
|
this.notificationClaims = this.readJson("notification-claims.json") ?? {};
|
|
21697
|
+
this.lastFleetRoll = this.readJson("last-fleet-roll.json") ?? null;
|
|
21518
21698
|
this.lastQuotaCache = this.readJson("last-quota.json") ?? {};
|
|
21519
21699
|
this.applyActiveOverride();
|
|
21520
21700
|
}
|
|
@@ -21572,6 +21752,10 @@ class AuthBroker {
|
|
|
21572
21752
|
persistNotificationClaims() {
|
|
21573
21753
|
atomicWriteJsonSync(join8(this.stateDir, "notification-claims.json"), this.notificationClaims, 384);
|
|
21574
21754
|
}
|
|
21755
|
+
persistLastFleetRoll() {
|
|
21756
|
+
if (this.lastFleetRoll)
|
|
21757
|
+
atomicWriteJsonSync(join8(this.stateDir, "last-fleet-roll.json"), this.lastFleetRoll, 384);
|
|
21758
|
+
}
|
|
21575
21759
|
persistShaIndex() {
|
|
21576
21760
|
atomicWriteJsonSync(join8(this.stateDir, "sha-index.json"), this.shaIndex, 384);
|
|
21577
21761
|
}
|
|
@@ -21608,7 +21792,8 @@ class AuthBroker {
|
|
|
21608
21792
|
accountKind: entry.accountKind,
|
|
21609
21793
|
ok: entry.ok,
|
|
21610
21794
|
error: entry.error,
|
|
21611
|
-
replace: entry.replace
|
|
21795
|
+
replace: entry.replace,
|
|
21796
|
+
reason: entry.reason
|
|
21612
21797
|
});
|
|
21613
21798
|
const auditPath = join8(this.stateDir, "audit.jsonl");
|
|
21614
21799
|
try {
|
|
@@ -12395,6 +12395,7 @@ var SwitchroomConfigSchema = exports_external.object({
|
|
|
12395
12395
|
uid: exports_external.number().int().nonnegative().optional().describe("Optional UID to chown the consumer socket to (defaults to 0 = root, " + "suitable for sibling containers running as root)."),
|
|
12396
12396
|
mirror_dir: exports_external.string().optional().describe("Optional host-side directory path. When set, the broker actively " + "writes the consumer's effective-account `.credentials.json` mirror " + "here \u2014 in addition to serving creds on demand via `get-credentials`. " + "Use this to eliminate the pull-latency gap: without a mirror the " + "consumer only gets failover creds at its next scheduled re-fetch " + "(up to 30 min). With a mirror the broker pushes failover creds " + "immediately when it detects exhaustion (consumer-quota-sensor tick, " + "or a mark-exhausted RPC on the pinned account). The directory must " + "be accessible to the broker container (bind-mounted from the host) " + "and to the consumer container; the broker writes " + "`<mirror_dir>/.credentials.json` atomically. Chown is attempted to " + "`uid` (default 0) \u2014 swallowed when CAP_CHOWN is absent.")
|
|
12397
12397
|
})).optional().describe("Non-agent peers that hold a broker socket (RFC H \u00a74.8). Each gets " + "its own `/run/switchroom/auth-broker/<name>/sock` chowned to its UID. " + "Consumers cannot be admins; a consumer name that collides with an " + "agent (whether that agent has `admin: true` or not) is a config " + "error caught at schema validation."),
|
|
12398
|
+
proactive_failover_pct: exports_external.number().min(1).max(99).optional().describe("Soft-avoid threshold (percent) for proactive auth failover (#3031). " + "When set (e.g. 95), an account whose fresh 7-day utilization is at/" + "above this value \u2014 or whose 5h utilization is at/above min(pct+3, " + "98) \u2014 enters the SOFT-AVOID preference tier: the broker prefers a " + "fully-eligible fallback account on the serving path, without ever " + "blocking the account (the hard wall stays 99.5). Enter/exit uses " + "hysteresis (exit below pct-5 or on window reset) so the preference " + "does not flap between probe ticks. Accounts serving past the wall " + "via `allow_overage_accounts` are never soft-avoided. UNSET (the " + "default) disables the tier entirely \u2014 behavior is identical to a " + "fleet without this field."),
|
|
12398
12399
|
allow_overage_accounts: exports_external.array(exports_external.string().min(1)).optional().describe("Opt-in list of account labels (bare strings matching `auth.active` / " + "`auth.fallback_order` entries) that may be served PAST the weekly " + "utilization wall when Anthropic overage billing is available for the " + "account (`overageStatus === 'allowed'`). Overage is REAL MONEY \u2014 " + "default is empty (no account gets this). An account in this list is " + "only kept eligible when its fresh quota snapshot reports " + "`overageStatus: 'allowed'` AND `overageDisabledReason` is NOT " + "'out_of_credits' (i.e. the overage credit has not been exhausted). " + "As soon as `overageDisabledReason` becomes 'out_of_credits', the " + "account is blocked immediately regardless of this flag. Overage lifts " + "ONLY the utilization wall \u2014 it cannot lift an active exhaustion mark " + "written by a real 429 (`mark-exhausted`).")
|
|
12399
12400
|
}).optional().describe("Switchroom-auth-broker configuration (RFC H). Fleet-wide active account, " + "fallback order, admin-agent ACL, and ephemeral-consumer surface. " + "Required from the v0.8+ schema onwards; pre-v0.8 fleets are migrated " + "in-place by `switchroom apply` (see src/auth/migrate-schema.ts)."),
|
|
12400
12401
|
drive: GoogleWorkspaceConfigSchema.describe("RFC D legacy key \u2014 use `google_workspace:` instead. Optional Google " + "Workspace onboarding configuration. When set, supplies Google OAuth " + "client credentials, the approver allowlist for `switchroom drive " + "connect`, and the optional tier knob. Env vars " + "(SWITCHROOM_GOOGLE_CLIENT_ID, SWITCHROOM_GOOGLE_CLIENT_SECRET, " + "SWITCHROOM_APPROVER_USER_ID) take precedence over this block when " + "set, preserving back-compat with the env-only flow shipped in #766."),
|