switchroom 0.20.8 → 0.20.9

package/telegram-plugin/tests/throttle-tier-probe-only.test.ts

@@ -0,0 +1,216 @@
+/**
+ * Tests for the generic-transient PROBE-ONLY 429 path (#failover-429-corroborate).
+ *
+ * This pins the OUTCOMES the gateway promises for a bare `rate_limit_error`
+ * (generic-transient) 429 after Ken picked Option A (probe-only):
+ *
+ *   - generic-transient + HEALTHY probe → the probe's ONLY effect is the
+ *     silent broker quota refresh. Account-inert at the runner: NO throttle
+ *     notice, NO self-restart nudge, NO `throttled_until` soft-defer, NO second
+ *     card. The calm rate-limited card the gateway already emitted stays the
+ *     ONLY user-visible output.
+ *   - generic-transient + WALL (probe corroborates) → the escalation path runs
+ *     exactly like the account-scoped `fire`: the corroborated-wall announcement
+ *     posts and the dead turn is resumed. No redundant calm card is emitted from
+ *     this branch — the announcement IS the output.
+ *   - litellm-local → the runner does NOT fire at all (account-inert by
+ *     mechanism, not discipline): the request never reached Anthropic, so
+ *     account state must not be touched. `account-scoped` likewise takes its own
+ *     `fire` path, not `fireProbeOnly`. Pinned via the pure classification gate
+ *     the gateway consults before calling `fireProbeOnly`.
+ *
+ * The runner side is exercised with fully injected deps (no gateway import),
+ * mirroring throttle-tier-wiring.test.ts.
+ */
+
+import { describe, it, expect } from 'vitest'
+import {
+  createThrottleTierRunner,
+  type ThrottleBrokerClient,
+  type ThrottleTierRunnerDeps,
+} from '../gateway/throttle-tier-wiring.js'
+import { classification429WarrantsCorroboration } from '../throttle-tier.js'
+
+const NOW = Date.UTC(2026, 6, 12, 8, 0, 0)
+
+interface Harness {
+  deps: ThrottleTierRunnerDeps
+  calls: {
+    markThrottled: Array<{ until: number; probeOnly?: boolean }>
+    claims: string[]
+    notices: Array<{ chatId: string | number; markdown: string }>
+    deferrals: string[]
+    restarts: string[]
+    logs: string[]
+    timers: Array<{ ms: number; fn: () => void; cancelled: boolean }>
+  }
+}
+
+function makeHarness(opts: {
+  markThrottledResult?:
+    | { account: string; throttled_until: number; escalated: boolean; rolledTo?: string | null }
+    | 'unreachable'
+    | 'throw'
+  turnInFlight?: () => boolean
+  newestTurnStartedAt?: () => number | null
+} = {}): Harness {
+  const nowMs = NOW
+  const calls: Harness['calls'] = {
+    markThrottled: [],
+    claims: [],
+    notices: [],
+    deferrals: [],
+    restarts: [],
+    logs: [],
+    timers: [],
+  }
+  const client: ThrottleBrokerClient = {
+    async markThrottled(until: number, probeOnly?: boolean) {
+      calls.markThrottled.push({ until, probeOnly })
+      if (opts.markThrottledResult === 'throw') throw new Error('boom')
+      const r = opts.markThrottledResult
+      if (r && r !== 'unreachable') return r
+      return { account: 'alice', throttled_until: until, escalated: false, rolledTo: null }
+    },
+    async claimNotification(key: string) {
+      calls.claims.push(key)
+      return { granted: true }
+    },
+  }
+  const deps: ThrottleTierRunnerDeps = {
+    agentName: 'carrie',
+    getBrokerClient: async () =>
+      opts.markThrottledResult === 'unreachable' ? null : client,
+    listNoticeChats: () => ['111', '222'],
+    sendNotice: (chatId, markdown) => calls.notices.push({ chatId, markdown }),
+    resumeDecide: () => 'resume',
+    newestActiveTurnStartedAtMs: opts.newestTurnStartedAt ?? (() => null),
+    turnInFlight: opts.turnInFlight ?? (() => false),
+    deferRestartToTurnComplete: (_agent, reason) => calls.deferrals.push(reason),
+    restartNow: (_agent, reason) => calls.restarts.push(reason),
+    log: (m) => calls.logs.push(m),
+    now: () => nowMs,
+    schedule: (fn, ms) => {
+      const t = { ms, fn, cancelled: false }
+      calls.timers.push(t)
+      return { cancel: () => { t.cancelled = true } }
+    },
+    jitterMs: () => 0,
+  }
+  return { deps, calls }
+}
+
+describe('generic-transient probe-only — classification gate (litellm-local never fires)', () => {
+  it('fires the corroboration probe ONLY for generic-transient', () => {
+    expect(classification429WarrantsCorroboration('generic-transient')).toBe(true)
+    // litellm-local: request never reached Anthropic — account-inert by mechanism.
+    expect(classification429WarrantsCorroboration('litellm-local')).toBe(false)
+    // account-scoped: runs its own throttle tier / failover (fire, not fireProbeOnly).
+    expect(classification429WarrantsCorroboration('account-scoped')).toBe(false)
+    // null: not a rate-limited event.
+    expect(classification429WarrantsCorroboration(null)).toBe(false)
+  })
+})
+
+describe('generic-transient probe-only — HEALTHY probe is account-inert', () => {
+  it('probes the broker but emits NOTHING else (calm card stays the only output)', async () => {
+    const h = makeHarness() // default result: escalated:false (healthy)
+    const runner = createThrottleTierRunner(h.deps)
+    await runner.fireProbeOnly('carrie')
+
+    // The probe reached the broker in probe-only mode…
+    expect(h.calls.markThrottled).toHaveLength(1)
+    expect(h.calls.markThrottled[0].probeOnly).toBe(true)
+
+    // …but produced NO user-visible or account-mutating side effects:
+    expect(h.calls.notices).toHaveLength(0) // no throttle notice, no second card
+    expect(h.calls.claims).toHaveLength(0) // no fleet-dedup broadcast at all
+    expect(h.calls.restarts).toHaveLength(0) // no self-restart nudge
+    expect(h.calls.deferrals).toHaveLength(0)
+    expect(h.calls.timers).toHaveLength(0) // no throttled_until soft-defer / retry nudge
+    expect(runner.inspect().nudgePending).toBe(false)
+  })
+
+  it('a broker-unreachable probe is inert and never throws', async () => {
+    const h = makeHarness({ markThrottledResult: 'unreachable' })
+    const runner = createThrottleTierRunner(h.deps)
+    await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
+    expect(h.calls.markThrottled).toHaveLength(0)
+    expect(h.calls.notices).toHaveLength(0)
+    expect(h.calls.restarts).toHaveLength(0)
+    expect(h.calls.timers).toHaveLength(0)
+  })
+
+  it('a markThrottled throw is swallowed and stays inert (no card, no restart)', async () => {
+    const h = makeHarness({ markThrottledResult: 'throw' })
+    const runner = createThrottleTierRunner(h.deps)
+    await expect(runner.fireProbeOnly('carrie')).resolves.toBeUndefined()
+    expect(h.calls.notices).toHaveLength(0)
+    expect(h.calls.restarts).toHaveLength(0)
+    expect(h.calls.timers).toHaveLength(0)
+  })
+})
+
+describe('generic-transient probe-only — WALL corroborated → escalation', () => {
+  it('posts the corroborated-wall announcement and resumes immediately (no calm card, no nudge timer)', async () => {
+    const h = makeHarness({
+      markThrottledResult: {
+        account: 'alice',
+        throttled_until: NOW + 60_000,
+        escalated: true,
+        rolledTo: 'bob',
+      },
+    })
+    const runner = createThrottleTierRunner(h.deps)
+    await runner.fireProbeOnly('carrie')
+
+    // Probe ran in probe-only mode, then escalation took over.
+    expect(h.calls.markThrottled[0].probeOnly).toBe(true)
+
+    // The escalation announcement (fleet-deduped), NOT the staying-put notice.
+    expect(h.calls.claims).toEqual([
+      `throttle-escalation:alice:111`,
+      `throttle-escalation:alice:222`,
+    ])
+    expect(h.calls.notices).toHaveLength(2)
+    expect(h.calls.notices[0].markdown).toContain('actually a wall')
+    expect(h.calls.notices[0].markdown).toContain('bob')
+
+    // Immediate resume via the escalation lever; NO delayed retry nudge armed.
+    expect(h.calls.restarts).toEqual(['throttle-escalation-resume'])
+    expect(h.calls.timers).toHaveLength(0)
+  })
+
+  it('escalated with rolledTo=null (all blocked) announces but does NOT restart', async () => {
+    const h = makeHarness({
+      markThrottledResult: {
+        account: 'alice',
+        throttled_until: NOW + 60_000,
+        escalated: true,
+        rolledTo: null,
+      },
+    })
+    const runner = createThrottleTierRunner(h.deps)
+    await runner.fireProbeOnly('carrie')
+    expect(h.calls.notices[0].markdown).toContain('all blocked')
+    expect(h.calls.restarts).toHaveLength(0)
+    expect(h.calls.timers).toHaveLength(0)
+  })
+
+  it('escalated resume respects the live-turn guards (defers under the dead turn gate)', async () => {
+    const h = makeHarness({
+      markThrottledResult: {
+        account: 'alice',
+        throttled_until: NOW + 60_000,
+        escalated: true,
+        rolledTo: 'bob',
+      },
+      turnInFlight: () => true,
+      newestTurnStartedAt: () => NOW - 60_000, // the dead turn still holds the gate
+    })
+    const runner = createThrottleTierRunner(h.deps)
+    await runner.fireProbeOnly('carrie')
+    expect(h.calls.restarts).toHaveLength(0)
+    expect(h.calls.deferrals).toEqual(['throttle-escalation-resume'])
+  })
+})

package/telegram-plugin/tests/throttle-tier-route-429-wiring.test.ts

@@ -0,0 +1,92 @@
+/**
+ * L1 SEAM-BOUNDARY GUARD for the 429 classify→route wiring (follow-up to
+ * switchroom#4381 / #4379).
+ *
+ * `throttle-tier-route-429.test.ts` proves the ROUTING LOGIC of the pure seam
+ * `routeRateLimit429` in isolation (generic-transient → fireProbeOnly;
+ * litellm-local / account-scoped / null → no probe). But nothing asserted that
+ * the GATEWAY still calls that seam at its 429-handling path. That is the exact
+ * gap the extraction opened: a future edit deleting the gateway callsite
+ * (gateway.ts, `emitGatewayOperatorEvent` rate-limited calm branch) would
+ * silently drop the corroboration probe for every `generic-transient` 429 —
+ * a real 5h/7d wall hiding behind transient wording would again die with no
+ * failover — while the seam's own unit suite stayed fully green.
+ *
+ * gateway.ts is a side-effecting IIFE that cannot be imported in a unit test
+ * (same constraint documented in turn-flush-suppression-wiring.test.ts /
+ * activity-card-wiring.test.ts): `emitGatewayOperatorEvent` is a module-scoped,
+ * un-exported function reachable only after the gateway boots. So an
+ * outcome-level "drive emitGatewayOperatorEvent and observe fireProbeOnly" test
+ * is infeasible without exporting internal gateway state — scope creep the
+ * repo deliberately avoids. Following the established wiring-guard pattern,
+ * this is a STRUCTURAL assertion that pins the load-bearing call site. It
+ * COMPLEMENTS the seam suite: the routing outcomes are proven there; this
+ * guards that the gateway actually routes through them.
+ *
+ * The outcome that flips on the regression: with the callsite present, a
+ * `generic-transient` 429 reaches `routeRateLimit429(..., throttleTierRunner,
+ * agent)` and fires the probe on the REAL runner; delete or neuter the
+ * callsite and these assertions go red. Verified fails-red by deleting
+ * gateway.ts:7831 locally (see PR body).
+ */
+import { describe, it, expect } from 'vitest'
+import { readFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+
+const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
+
+function between(src: string, startMarker: string, endMarker: string): string {
+  const after = src.split(startMarker)[1] ?? ''
+  return after.split(endMarker)[0] ?? ''
+}
+
+/** Strip comment lines so prose (a docstring mentioning the call, or a comment
+ *  describing the OLD shape) can neither satisfy nor trip an assertion about
+ *  the actual CODE. */
+function codeOnly(src: string): string {
+  return src
+    .split('\n')
+    .filter((l) => !l.trim().startsWith('//'))
+    .filter((l) => !l.trim().startsWith('*') && !l.trim().startsWith('/*'))
+    .join('\n')
+}
+
+describe('429 route-seam wiring — the gateway calls routeRateLimit429 (switchroom#4381 L1)', () => {
+  // The calm-429 branch: from the classification binding through the surface
+  // decision that immediately follows the seam call.
+  const branch = between(
+    gatewaySrc,
+    'const rateLimit429Classification =',
+    "if (surface === 'litellm-local-notice')",
+  )
+
+  it('the calm-429 branch routes the classification through routeRateLimit429', () => {
+    expect(branch.length).toBeGreaterThan(100)
+    const code = codeOnly(branch)
+    // The load-bearing call site. Deleting it drops the corroboration probe for
+    // every generic-transient 429 with the seam's unit suite still green — the
+    // exact regression this guard exists to catch.
+    expect(code).toMatch(/routeRateLimit429\(/)
+    // It must be fed the CLASSIFICATION computed for this event, not a literal
+    // or a stale variable — otherwise the wrong family would (not) be probed.
+    expect(code).toMatch(
+      /routeRateLimit429\(\s*rateLimit429Classification\s*,/,
+    )
+  })
+
+  it('the seam is handed the REAL throttleTierRunner (a live probe, not a no-op)', () => {
+    const code = codeOnly(branch)
+    // Passing anything other than the gateway's real runner (e.g. a stub) would
+    // make the probe fire into the void — the branch outcome would look correct
+    // in a naive grep but corroborate nothing.
+    expect(code).toMatch(
+      /routeRateLimit429\(\s*rateLimit429Classification\s*,\s*throttleTierRunner\s*,\s*agent\s*\)/,
+    )
+  })
+
+  it('routeRateLimit429 is imported into the gateway (the seam is actually linked)', () => {
+    // A dangling call to an unimported symbol would fail tsc, but pinning the
+    // import makes the dependency edge explicit and its removal a red test too.
+    expect(codeOnly(gatewaySrc)).toMatch(/\brouteRateLimit429\b/)
+  })
+})

package/telegram-plugin/tests/throttle-tier-route-429.test.ts

@@ -0,0 +1,71 @@
+/**
+ * Tests for the classify→route WIRING of the gateway's 429 corroboration path
+ * (#failover-429-corroborate, follow-up switchroom#4379).
+ *
+ * The pure gate `classification429WarrantsCorroboration` is already unit-tested
+ * (throttle-tier-probe-only.test.ts), and the runner's `fireProbeOnly` outcomes
+ * are pinned there too. What was NOT covered is the WIRING that connects them:
+ * the gateway callsite (gateway.ts, `handleOperatorEvent` rate-limited branch)
+ * that decides whether to invoke `throttleTierRunner.fireProbeOnly(agent)` for a
+ * given classification.
+ *
+ * gateway.ts is hard to unit-test in isolation and is under a hard line ratchet,
+ * so rather than stand up a gateway harness the decision was extracted into a
+ * pure, injectable seam — `routeRateLimit429(classification, runner, agent)` in
+ * throttle-tier.ts — which the gateway callsite now calls verbatim. These tests
+ * exercise that seam with a fake runner that records which entrypoint fired,
+ * pinning the three routing outcomes the gateway promises:
+ *
+ *   - generic-transient → routes to fireProbeOnly (the probe fires)
+ *   - litellm-local     → does NOT route to fireProbeOnly (account-inert by
+ *                         mechanism: the request never reached Anthropic)
+ *   - account-scoped    → does NOT route to fireProbeOnly (that classification
+ *                         takes its own `fire` path, not the probe)
+ */
+
+import { describe, it, expect } from 'vitest'
+import { routeRateLimit429, type RateLimit429ProbeRunner } from '../throttle-tier.js'
+
+interface FakeRunner extends RateLimit429ProbeRunner {
+  probeCalls: string[]
+}
+
+function makeFakeRunner(): FakeRunner {
+  const probeCalls: string[] = []
+  return {
+    probeCalls,
+    async fireProbeOnly(triggerAgent: string) {
+      probeCalls.push(triggerAgent)
+    },
+  }
+}
+
+describe('routeRateLimit429 — gateway classify→route wiring (switchroom#4379)', () => {
+  it('generic-transient → routes to fireProbeOnly', () => {
+    const runner = makeFakeRunner()
+    const fired = routeRateLimit429('generic-transient', runner, 'carrie')
+    expect(fired).toBe(true)
+    expect(runner.probeCalls).toEqual(['carrie'])
+  })
+
+  it('litellm-local → does NOT route to fireProbeOnly (stays account-inert)', () => {
+    const runner = makeFakeRunner()
+    const fired = routeRateLimit429('litellm-local', runner, 'carrie')
+    expect(fired).toBe(false)
+    expect(runner.probeCalls).toHaveLength(0)
+  })
+
+  it('account-scoped → does NOT route to fireProbeOnly (uses fire, not the probe)', () => {
+    const runner = makeFakeRunner()
+    const fired = routeRateLimit429('account-scoped', runner, 'carrie')
+    expect(fired).toBe(false)
+    expect(runner.probeCalls).toHaveLength(0)
+  })
+
+  it('null (not a rate-limited event) → does NOT route to fireProbeOnly', () => {
+    const runner = makeFakeRunner()
+    const fired = routeRateLimit429(null, runner, 'carrie')
+    expect(fired).toBe(false)
+    expect(runner.probeCalls).toHaveLength(0)
+  })
+})

package/telegram-plugin/throttle-tier.ts

@@ -120,6 +120,65 @@ export function classify429Detail(text: string): RateLimit429Classification {
   return 'generic-transient'
 }
 
+/**
+ * #failover-429-corroborate — does this terminal 429 classification warrant a
+ * fire-and-forget broker corroboration probe (throttle-tier runner's
+ * PROBE-ONLY entrypoint) IN ADDITION to the calm rate-limited card?
+ *
+ * TRUE for `generic-transient` ONLY. That family (a bare `rate_limit_error`
+ * body / novel wording that matches neither the account-scoped negation
+ * strings nor litellm-local) used to drop on the calm-card floor with NO broker
+ * contact — so a genuine 5h/7d wall hiding behind transient wording was never
+ * probed, and the turn died with no failover. Routing it through the runner's
+ * `fireProbeOnly` lets the broker take ONE live quota probe (rate-bounded) and
+ * convert a real wall into failover. A HEALTHY probe is fully account-inert:
+ * probe-only records NO `throttled_until`, sends NO throttle notice, arms NO
+ * self-restart nudge, and adds NO second card — so the calm rate-limited card
+ * stays the ONLY user-visible output, exactly as before this path existed.
+ *
+ * FALSE (never corroborate) for:
+ *   - `litellm-local` — INVARIANT: the request never reached Anthropic (the
+ *     proxy's own tpm/rpm limiter tripped), so account state must not be
+ *     touched. This mechanism, not discipline, keeps that path account-inert.
+ *   - `account-scoped` — already runs its own throttle tier / failover path.
+ *   - `null` — not a rate-limited event.
+ */
+export function classification429WarrantsCorroboration(
+  classification: RateLimit429Classification | null,
+): boolean {
+  return classification === 'generic-transient'
+}
+
+/**
+ * Minimal runner shape the classify→route seam needs: just the PROBE-ONLY
+ * entrypoint. The full gateway `ThrottleTierRunner` satisfies it structurally,
+ * and a test can supply a fake that records the call.
+ */
+export interface RateLimit429ProbeRunner {
+  fireProbeOnly(triggerAgent: string): Promise<void>
+}
+
+/**
+ * #failover-429-corroborate — the classify→route WIRING extracted from the
+ * gateway callsite so the routing decision (not just the gate) is unit-testable
+ * without a gateway harness. Fires the runner's PROBE-ONLY entrypoint IFF the
+ * classification warrants corroboration (`generic-transient` only, per
+ * `classification429WarrantsCorroboration`); `litellm-local`, `account-scoped`,
+ * and `null` never fire it. Fire-and-forget: the runner promise is intentionally
+ * not awaited (the gateway must not block the operator-event path on a probe).
+ * Returns `true` when a probe was fired, `false` otherwise — observable in tests
+ * and unused at the gateway callsite.
+ */
+export function routeRateLimit429(
+  classification: RateLimit429Classification | null,
+  runner: RateLimit429ProbeRunner,
+  agent: string,
+): boolean {
+  if (!classification429WarrantsCorroboration(classification)) return false
+  void runner.fireProbeOnly(agent)
+  return true
+}
+
 /**
  * Build the `rate_limit_429_classified` runtime metric for one terminal
  * rate-limited operator event — the instrumentation that lets an operator